###############################################################################
# NMDS_bioenv
#------------------------------------------------------------------------------
# by Umer Zeeshan Ijaz (http://userweb.eng.gla.ac.uk/umer.ijaz)
#
###############################################################################

#==============================================================================
# LIBRARY DEPENDENCE
#==============================================================================
library(vegan)
library(ggplot2)
library(grid)

#==============================================================================
# Let's rock'n'roll!
#==============================================================================
#
# Tutorial on plotting significant taxa and environmental variables on an NMDS
# plot using ggplot2.
# by Umer Zeeshan Ijaz (http://userweb.eng.gla.ac.uk/umer.ijaz)
#

# This R script is an extension of vegan library's bioenv()
# function and uses the bio.env() and bio.step() of
#	http://menugget.blogspot.co.uk/2011/06/clarke-and-ainsworths-bioenv-and-bvstep.html
#	The original author suggested these functions to overcome
#	the inflexibility of the bioenv() function which uses
#	a similarity matrix based on normalized "euclidean" distance.
# The new functions are given below and implement the following algorithms:
# Clarke, K. R & Ainsworth, M. 1993. A method of linking multivariate community structure to environmental variables. Marine Ecology Progress Series, 92, 205-219.
# Clarke, K. R., Gorley, R. N., 2001. PRIMER v5: User Manual/Tutorial. PRIMER-E, Plymouth, UK.
# Clarke, K. R., Warwick, R. M., 2001. Changes in Marine Communities: An Approach to Statistical Analysis and Interpretation, 2nd edition. PRIMER-E Ltd, Plymouth, UK.
# Clarke, K. R., Warwick, R. M., 1998. Quantifying structural redundancy in ecological communities. Oecologia, 113:278-289.

bv.step  <-  function(fix.mat, var.mat,
                    fix.dist.method="bray", var.dist.method="euclidean", correlation.method="spearman",
                    scale.fix=FALSE, scale.var=TRUE,
                    max.rho=0.95,
                    min.delta.rho=0.001,
                    random.selection=TRUE,
                    prop.selected.var=0.2,
                    num.restarts=10,
                    var.always.include=NULL,
                    var.exclude=NULL,
                    output.best=10
){

  if(dim(fix.mat)[1] != dim(var.mat)[1]){stop("fixed and variable matrices must have the same number of rows")}
  if(sum(var.always.include %in% var.exclude) > 0){stop("var.always.include and var.exclude share a variable")}
  require(vegan)

  if(scale.fix){fix.mat <- scale(fix.mat)}else{fix.mat <- fix.mat}
  if(scale.var){var.mat <- scale(var.mat)}else{var.mat <- var.mat}

  fix.dist  <-  vegdist(as.matrix(fix.mat), method=fix.dist.method)

  # an initial removal phase
  var.dist.full  <-  vegdist(as.matrix(var.mat), method=var.dist.method)
  full.cor  <-  suppressWarnings(cor.test(fix.dist, var.dist.full, method=correlation.method))$estimate
  var.comb  <-  combn(1:ncol(var.mat), ncol(var.mat)-1)
  RES  <-  data.frame(var.excl=rep(NA,ncol(var.comb)), n.var=ncol(var.mat)-1, rho=NA)
  for(i in 1:dim(var.comb)[2]){
    var.dist  <-  vegdist(as.matrix(var.mat[,var.comb[,i]]), method=var.dist.method)
    temp  <-  suppressWarnings(cor.test(fix.dist, var.dist, method=correlation.method))
    RES$var.excl[i]  <-  c(1:ncol(var.mat))[-var.comb[,i]]
    RES$rho[i]  <-  temp$estimate
  }
  delta.rho  <-  RES$rho - full.cor
  exclude  <-  sort(unique(c(RES$var.excl[which(abs(delta.rho) < min.delta.rho)], var.exclude)))

  if(random.selection){
    num.restarts=num.restarts
    prop.selected.var=prop.selected.var
    prob <- rep(1,ncol(var.mat))
    if(prop.selected.var< 1){
      prob[exclude] <- 0
    }
    n.selected.var  <-  min(sum(prob),prop.selected.var*dim(var.mat)[2])
  } else {
    num.restarts=1
    prop.selected.var=1
    prob <- rep(1,ncol(var.mat))
    n.selected.var  <-  min(sum(prob),prop.selected.var*dim(var.mat)[2])
  }

  RES_TOT  <-  c()
  for(i in 1:num.restarts){
    step=1
    RES  <-  data.frame(step=step, step.dir="F", var.incl=NA, n.var=0, rho=0)
    attr(RES$step.dir, "levels")  <-  c("F","B")
    best.comb  <-  which.max(RES$rho)
    best.rho  <-  RES$rho[best.comb]
    delta.rho  <-  Inf
    selected.var  <-  sort(unique(c(sample(1:dim(var.mat)[2], n.selected.var, prob=prob), var.always.include)))
    while(best.rho < max.rho & delta.rho > min.delta.rho & RES$n.var[best.comb] < length(selected.var)){
      # forward step
      step.dir="F"
      step=step+1
      var.comb  <-  combn(selected.var, RES$n.var[best.comb]+1, simplify=FALSE)
      if(RES$n.var[best.comb] == 0){
        var.comb.incl <- 1:length(var.comb)
      } else {
        var.keep  <-  as.numeric(unlist(strsplit(RES$var.incl[best.comb], ",")))
        temp  <-  NA*1:length(var.comb)
        for(j in 1:length(temp)){
          temp[j]  <-  all(var.keep %in% var.comb[[j]])
        }
        var.comb.incl  <-  which(temp==1)
      }

      RES.f  <-  data.frame(step=rep(step, length(var.comb.incl)), step.dir=step.dir, var.incl=NA, n.var=RES$n.var[best.comb]+1, rho=NA)
      for(f in 1:length(var.comb.incl)){
        var.incl  <-  var.comb[[var.comb.incl[f]]]
        var.incl  <-  var.incl[order(var.incl)]
        var.dist  <-  vegdist(as.matrix(var.mat[,var.incl]), method=var.dist.method)
        temp  <-  suppressWarnings(cor.test(fix.dist, var.dist, method=correlation.method))
        RES.f$var.incl[f]  <-  paste(var.incl, collapse=",")
        RES.f$rho[f]  <-  temp$estimate
      }

      last.F  <-  max(which(RES$step.dir=="F"))
      RES  <-  rbind(RES, RES.f[which.max(RES.f$rho),])
      best.comb  <-  which.max(RES$rho)
      delta.rho  <-  RES$rho[best.comb] - best.rho
      best.rho  <-  RES$rho[best.comb]

      if(best.comb == step){
        while(best.comb == step & RES$n.var[best.comb] > 1){
          # backward step
          step.dir="B"
          step  <-  step+1
          var.keep  <-  as.numeric(unlist(strsplit(RES$var.incl[best.comb], ",")))
          var.comb  <-  combn(var.keep, RES$n.var[best.comb]-1, simplify=FALSE)
          RES.b  <-  data.frame(step=rep(step, length(var.comb)), step.dir=step.dir, var.incl=NA, n.var=RES$n.var[best.comb]-1, rho=NA)
          for(b in 1:length(var.comb)){
            var.incl  <-  var.comb[[b]]
            var.incl  <-  var.incl[order(var.incl)]
            var.dist  <-  vegdist(as.matrix(var.mat[,var.incl]), method=var.dist.method)
            temp  <-  suppressWarnings(cor.test(fix.dist, var.dist, method=correlation.method))
            RES.b$var.incl[b]  <-  paste(var.incl, collapse=",")
            RES.b$rho[b]  <-  temp$estimate
          }
          RES  <-  rbind(RES, RES.b[which.max(RES.b$rho),])
          best.comb  <-  which.max(RES$rho)
          best.rho <-  RES$rho[best.comb]
        }
      } else {
        break()
      }

    }

    RES_TOT  <-  rbind(RES_TOT, RES[2:dim(RES)[1],])
    print(paste(round((i/num.restarts)*100,3), "% finished"))
  }

  RES_TOT  <-  unique(RES_TOT[,3:5])

  if(dim(RES_TOT)[1] > output.best){
    order.by.best  <-  RES_TOT[order(RES_TOT$rho, decreasing=TRUE)[1:output.best],]
  } else {
    order.by.best  <-   RES_TOT[order(RES_TOT$rho, decreasing=TRUE), ]
  }
  rownames(order.by.best) <- NULL

  order.by.i.comb  <-  c()
  for(i in 1:length(selected.var)){
    f1  <-  which(RES_TOT$n.var==i)
    f2  <-  which.max(RES_TOT$rho[f1])
    order.by.i.comb  <-  rbind(order.by.i.comb, RES_TOT[f1[f2],])
  }
  rownames(order.by.i.comb) <- NULL

  if(length(exclude)<1){var.exclude=NULL} else {var.exclude=exclude}
  out  <-  list(
    order.by.best=order.by.best,
    order.by.i.comb=order.by.i.comb,
    best.model.vars=paste(colnames(var.mat)[as.numeric(unlist(strsplit(order.by.best$var.incl[1], ",")))], collapse=","),
    best.model.rho=order.by.best$rho[1],
    var.always.include=var.always.include,
    var.exclude=var.exclude
  )
  out

}

bio.env  <-  function(fix.mat, var.mat,
                    fix.dist.method="bray", var.dist.method="euclidean", correlation.method="spearman",
                    scale.fix=FALSE, scale.var=TRUE,
                    output.best=10,
                    var.max=ncol(var.mat)
){
  if(dim(fix.mat)[1] != dim(var.mat)[1]){stop("fixed and variable matrices must have the same number of rows")}
  if(var.max > dim(var.mat)[2]){stop("var.max cannot be larger than the number of variables (columns) in var.mat")}

  require(vegan)

  combn.sum  <-  sum(factorial(ncol(var.mat))/(factorial(1:var.max)*factorial(ncol(var.mat)-1:var.max)))

  if(scale.fix){fix.mat <- scale(fix.mat)}else{fix.mat <- fix.mat}
  if(scale.var){var.mat <- scale(var.mat)}else{var.mat <- var.mat}
  fix.dist  <-  vegdist(fix.mat, method=fix.dist.method)
  RES_TOT  <-  c()
  best.i.comb  <-  c()
  iter  <-  0
  for(i in 1:var.max){
    var.comb  <-  combn(1:ncol(var.mat), i, simplify=FALSE)
    RES  <-  data.frame(var.incl=rep(NA, length(var.comb)), n.var=i, rho=0)
    for(f in 1:length(var.comb)){
      iter  <-  iter+1
      var.dist  <-  vegdist(as.matrix(var.mat[,var.comb[[f]]]), method=var.dist.method)
      temp  <-  suppressWarnings(cor.test(fix.dist, var.dist, method=correlation.method))
      RES$var.incl[f]  <-  paste(var.comb[[f]], collapse=",")
      RES$rho[f]  <-  temp$estimate
      if(iter %% 100 == 0){print(paste(round(iter/combn.sum*100, 3), "% finished"))}
    }

    order.rho  <-  order(RES$rho, decreasing=TRUE)
    best.i.comb  <-  c(best.i.comb, RES$var.incl[order.rho[1]])
    if(length(order.rho) > output.best){
      RES_TOT  <-  rbind(RES_TOT, RES[order.rho[1:output.best],])
    } else {
      RES_TOT  <-  rbind(RES_TOT, RES)
    }
  }
  rownames(RES_TOT) <- NULL

  if(dim(RES_TOT)[1] > output.best){
    order.by.best  <-  order(RES_TOT$rho, decreasing=TRUE)[1:output.best]
  } else {
    order.by.best  <-  order(RES_TOT$rho, decreasing=TRUE)
  }
  OBB  <-  RES_TOT[order.by.best,]
  rownames(OBB)  <-  NULL

  order.by.i.comb  <-  match(best.i.comb, RES_TOT$var.incl)
  OBC  <-  RES_TOT[order.by.i.comb,]
  rownames(OBC)  <-  NULL

  out  <-  list(
    order.by.best=OBB,
    order.by.i.comb=OBC,
    best.model.vars=paste(colnames(var.mat)[as.numeric(unlist(strsplit(OBB$var.incl[1], ",")))], collapse=",") ,
    best.model.rho=OBB$rho[1]
  )
  out
}

abund_table <- read.csv("SPE_pitlatrine.csv",row.names=1,check.names=FALSE)

# Transpose the data to have sample names on rows
abund_table <- t(abund_table)
meta_table <- read.csv("ENV_pitlatrine.csv",row.names=1,check.names=FALSE)

# Just a check to ensure that the samples in meta_table are in the same order as in abund_table
meta_table <- meta_table[rownames(abund_table),]

# Get grouping information
grouping_info <- data.frame(row.names=rownames(abund_table),t(as.data.frame(strsplit(rownames(abund_table),"_"))))

# Parameters
cmethod <- "pearson" # Correlation method to use: pearson, pearman, kendall
fmethod <- "bray" # Fixed distance method: euclidean, manhattan, gower, altGower, canberra, bray, kulczynski, morisita,horn, binomial, and cao
vmethod <- "bray" # Variable distance method: euclidean, manhattan, gower, altGower, canberra, bray, kulczynski, morisita,horn, binomial, and cao
nmethod <- "bray" # NMDS distance method:  euclidean, manhattan, gower, altGower, canberra, bray, kulczynski, morisita,horn, binomial, and cao

res  <-  bio.env(wisconsin(abund_table), meta_table,fix.dist.method=fmethod, var.dist.method=vmethod, correlation.method=cmethod,
               scale.fix=FALSE, scale.var=TRUE)

# Get the 10 best subset of environmental variables
envNames <- colnames(meta_table)
bestEnvFit <- ""
for(i in (1:length(res$order.by.best$var.incl)))
{
  bestEnvFit[i] <- paste(paste(envNames[as.numeric(unlist(strsplit(res$order.by.best$var.incl[i], split=",")))],collapse=' + '), " = ",res$order.by.best$rho[i],sep="")
}
bestEnvFit <- data.frame(bestEnvFit)
colnames(bestEnvFit) <- "Best combination of environmental variables with similarity score"

res.bv.step.biobio  <-  bv.step(wisconsin(abund_table), wisconsin(abund_table),
                              fix.dist.method=fmethod, var.dist.method=vmethod,correlation.method=cmethod,
                              scale.fix=FALSE, scale.var=FALSE,
                              max.rho=0.95, min.delta.rho=0.001,
                              random.selection=TRUE,
                              prop.selected.var=0.3,
                              num.restarts=10,
                              output.best=10,
                              var.always.include=NULL)

# Get the 10 best subset of taxa
taxaNames <- colnames(abund_table)
bestTaxaFit <- ""
for(i in (1:length(res.bv.step.biobio$order.by.best$var.incl)))
{
  bestTaxaFit[i] <- paste(paste(taxaNames[as.numeric(unlist(strsplit(res.bv.step.biobio$order.by.best$var.incl[i], split=",")))],collapse=' + '), " = ",res.bv.step.biobio$order.by.best$rho[i],sep="")
}
bestTaxaFit <- data.frame(bestTaxaFit)
colnames(bestTaxaFit) <- "Best combination of taxa with similarity score"

# > bestTaxaFit
# Best combination of taxa with similarity score
# 1                             Bacilli + Bacteroidia + Chrysiogenetes + Clostridia + Dehalococcoidetes + Fibrobacteria + Flavobacteria + Fusobacteria + Gammaproteobacteria + Methanomicrobia + Mollicutes + Opitutae + Synergistia + Thermomicrobia + Unknown = 0.900998476657462
# 2                                              Bacilli + Bacteroidia + Clostridia + Dehalococcoidetes + Fibrobacteria + Flavobacteria + Fusobacteria + Gammaproteobacteria + Methanomicrobia + Mollicutes + Opitutae + Synergistia + Thermomicrobia + Unknown = 0.899912718316228
# 3                                                        Bacilli + Bacteroidia + Clostridia + Dehalococcoidetes + Fibrobacteria + Flavobacteria + Fusobacteria + Gammaproteobacteria + Methanomicrobia + Mollicutes + Opitutae + Synergistia + Thermomicrobia = 0.896821772937576
# 4  Clostridia + Dehalococcoidetes + Deltaproteobacteria + Epsilonproteobacteria + Erysipelotrichi + Flavobacteria + Fusobacteria + Methanobacteria + Mollicutes + Opitutae + Planctomycetacia + Sphingobacteria + Subdivision3 + Synergistia + Thermomicrobia = 0.892670058822226
# 5                                                                     Bacilli + Bacteroidia + Clostridia + Dehalococcoidetes + Fibrobacteria + Flavobacteria + Fusobacteria + Gammaproteobacteria + Methanomicrobia + Opitutae + Synergistia + Thermomicrobia = 0.892533063335985
# 6                   Clostridia + Dehalococcoidetes + Deltaproteobacteria + Epsilonproteobacteria + Erysipelotrichi + Flavobacteria + Fusobacteria + Methanobacteria + Mollicutes + Opitutae + Planctomycetacia + Sphingobacteria + Subdivision3 + Synergistia = 0.891217789278463
# 7                                     Clostridia + Dehalococcoidetes + Deltaproteobacteria + Epsilonproteobacteria + Erysipelotrichi + Flavobacteria + Fusobacteria + Mollicutes + Opitutae + Planctomycetacia + Sphingobacteria + Subdivision3 + Synergistia = 0.888669881927483
# 8                                                                                Bacilli + Bacteroidia + Clostridia + Dehalococcoidetes + Fibrobacteria + Flavobacteria + Fusobacteria + Gammaproteobacteria + Opitutae + Synergistia + Thermomicrobia = 0.887052815492516
# 9                                                  Clostridia + Dehalococcoidetes + Deltaproteobacteria + Epsilonproteobacteria + Erysipelotrichi + Flavobacteria + Fusobacteria + Opitutae + Planctomycetacia + Sphingobacteria + Subdivision3 + Synergistia = 0.885880090785632
# 10                             Acidobacteria_Gp4 + Actinobacteria + Bacilli + Betaproteobacteria + Clostridia + Dehalococcoidetes + Erysipelotrichi + Fibrobacteria + Lentisphaeria + Methanomicrobia + Opitutae + Sphingobacteria + Thermomicrobia + Unknown = 0.882956559638505

# Generate NMDS plot
MDS_res=metaMDS(abund_table, distance = nmethod, k = 2, trymax = 50)

bio.keep  <-  as.numeric(unlist(strsplit(res.bv.step.biobio$order.by.best$var.incl[1], ",")))
bio.fit  <-  envfit(MDS_res, abund_table[,bio.keep,drop=F], perm = 999)

# use the best set of environmental variables in env.keep
eval(parse(text=paste("env.keep  <-  c(",res$order.by.best$var.incl[1],")",sep="")))
env.fit  <-  envfit(MDS_res, meta_table[,env.keep,drop=F], perm = 999)

# Get site information
df <- scores(MDS_res,display=c("sites"))

# Add grouping information
df <- data.frame(df,Type=grouping_info[rownames(df),1])

# Get the vectors for bioenv.fit
df_biofit <- scores(bio.fit,display=c("vectors"))
df_biofit <- df_biofit*vegan:::ordiArrowMul(df_biofit)
df_biofit <- as.data.frame(df_biofit)

# Get the vectors for env.fit
df_envfit <- scores(env.fit,display=c("vectors"))
df_envfit <- df_envfit*vegan:::ordiArrowMul(df_envfit)
df_envfit <- as.data.frame(df_envfit)

#==============================================================================
# Draw samples
#==============================================================================

p <- ggplot()
p <- p+geom_point(data=df,aes(NMDS1,NMDS2,colour=Type))
# Draw taxas
p <- p+geom_segment(data=df_biofit, aes(x = 0, y = 0, xend = NMDS1, yend = NMDS2),
                  arrow = arrow(length = unit(0.2, "cm")),color="# 808080",alpha=0.5)

p <- p+geom_text(data=as.data.frame(df_biofit*1.1),aes(NMDS1, NMDS2, label = rownames(df_biofit)),color="# 808080",alpha=0.5)
# Draw environmental variables
p <- p+geom_segment(data=df_envfit, aes(x = 0, y = 0, xend = NMDS1, yend = NMDS2),
                  arrow = arrow(length = unit(0.2, "cm")),color="# 4C005C",alpha=0.5)

p <- p+geom_text(data=as.data.frame(df_envfit*1.1),aes(NMDS1, NMDS2, label = rownames(df_envfit)),color="# 4C005C",alpha=0.5)
p <- p+theme_bw()
pdf("NMDS_bioenv.pdf")
print(p)
dev.off()


#' @title calcKristWettedSA
#'
#'@description Calculate hull wetted surface area (\code{wettedSA}) (m^2)
#' using the Kristensen method.
#'
#' @param shipType Ship type (vector of strings, see \code{\link{calcShipType}}).
#' Must align with \code{paxTugShipTypes},
#'\code{tankerBulkCarrierGCargoShipTypes}, and \code{containerShipTypes}
#'groupings
#'@param maxDisplacement Maximum ship displacement (vector of numericals, m^3)
#'@param maxDraft Maximum summer load line draft (vector of numericals, m)
#'@param actualDraft Actual draft (vector of numericals, m)
#'@param lwl Waterline length (vector of numericals, m) (see \code{\link{calclwl}})
#'@param breadth Moulded breadth (vector of numericals, m)
#'@param Cbw Waterline block coefficient (vector of numericals, dimensionless)
#'(see \code{\link{calcCbw}})
#'@param seawaterDensity Sea water density. Default = 1.025 (g/cm^3). Can
#' supply either a vector of numericals, a single number, or rely on the default
#'@param paxTugShipTypes Ship types specified in input \code{shipTypes} to be
#'modeled as passengers and tug vessels (vector of strings)
#'@param tankerBulkCarrierGCargoShipTypes Ship types specified in input
#'\code{shipTypes} to be modeled as tankers, bulk carriers and general cargo
#'vessels (vector of strings)
#'@param containerShipTypes Ship types specified in input \code{shipTypes}to be
#'modeled as container ships (vector of strings)
#'
#' @details
#'
#' This method this requires ship types to be grouped. Use the
#' \code{paxTugShipTypes}, \code{tankerBulkCarrierGCargoShipTypes}, and
#' \code{containerShipTypes} grouping  parameters to provide these ship
#' type groupings. Any ship types not included in this grouping will be
#' considered as miscellaneous vessels.
#'
#'
#' Container ship wetted surface area:
#'
#' = 0.995(disp./lloyds draft + 1.9 * Lwl * lloyds draft)-2.4(lloyds draft-actual draft)(lwl+breadth)
#'
#' Bulk carrier, tanker, and general cargo ship wetted surface area:
#'
#' = 0.99(disp./lloyds draft + 1.9 * Lwl * lloyds draft)-2(lloyds draft- actual draft)(lwl+breadth)
#'
#' Cruise ships and tugs treated as twin screw RORO in Kristensen formulas on
#' the basis of sample of vessels from Lloyds and their number of propellers.
#'
#' Twin Screw RORO wetted surface area:
#'
#' = 1.21(disp./lloyds draft +1.3*lwl*lloyds draft)*(1.2-0.34*Cbw)-2.5(lloyds draft-actual draft)*(lwl+breadth)
#'
#' Auto carriers, miscellaneous, reefer, and RORO ships are treated as single
#' skeg ROROs in Kristensen formulas on the basis of sample of vessels from
#' Lloyds and their number of propellers.
#'
#' Single screw RORO wetted surface area:
#'
#' 0.87(disp./lloyds draft +2.7*lwl*lloyds draft)*(1.2-0.34*Cbw)-3.0(lloyds draft-actual draft)*(lwl+breadth)
#'
#' Note: Actual draft is typically obtained from sources such as AIS messages or ship records.
#'
#' @return \code{wettedSA} (vector of numericals, m^2)
#'
#' @references
#'Kristensen, H. O. and Lutzen, M. 2013. "Prediction of Resistance and Propulsion
#'Power of Ships."
#'
#'\href{https://gitlab.gbar.dtu.dk/oceanwave3d/Ship-Desmo}{Kristensen, H. O.
#'"Ship-Desmo-Tool." https://gitlab.gbar.dtu.dk/oceanwave3d/Ship-Desmo}
#'
#'@seealso \itemize{
#'\item \code{\link{calclwl}}
#'\item \code{\link{calcCbw}}
#'\item \code{\link{calcShipType}}
#'}
#'
#' @family Kristensen Calculations
#'
#' @examples
#' calcKristWettedSA("bulk.carrier", 80097, 13.6, 12.5, 218, 32.25, 0.8099003, 1.025)
#' calcKristWettedSA("other.tanker", 238942, 15.6, 14, 400, 49, 0.7490287, 1.025,
#'                   tankerBulkCarrierGCargoShipTypes=c("other.tanker","bulk.carrier"))
#' calcKristWettedSA("container.ship", 238942, 15.6, 14, 400, 49, 0.7490287, 1.025)
#'
#' @export

calcKristWettedSA <- function(shipType, maxDisplacement, maxDraft, actualDraft, lwl, breadth, Cbw, seawaterDensity=1.025,
                              tankerBulkCarrierGCargoShipTypes=c("general.cargo","tanker","chemical.tanker","liquified.gas.tanker","oil.tanker","other.tanker","bulk.carrier"),
                              containerShipTypes=c("container.ship"),
                              paxTugShipTypes=c("ferry.pax","ferry.ro.pax","cruise","cruise.ed","yacht","tug","passenger","ro.ro")
                              ){

  wettedSA<- ifelse(#case 1
    shipType %in% containerShipTypes,
                    0.995*( maxDisplacement/(seawaterDensity*maxDraft) + 1.9*lwl*maxDraft)
                    - 2.4*(maxDraft-actualDraft)*(lwl+breadth)
                    ,
                    ifelse(#case2
                      shipType %in% tankerBulkCarrierGCargoShipTypes,
                            0.99*( maxDisplacement/(seawaterDensity*maxDraft) + 1.9 *lwl*maxDraft)
                            - 2*(maxDraft-actualDraft)*(lwl+breadth)
                            ,
                            ifelse(#case3
                              shipType %in% paxTugShipTypes,
                                  1.21*( maxDisplacement/(seawaterDensity*maxDraft) + 1.3*lwl*maxDraft)*(1.2-0.34*Cbw)
                                  - 2.5*(maxDraft-actualDraft)*(lwl+breadth)
                                  ,
                                  #miscelaneous case: reefer, roro, etc...
                                  0.87*( maxDisplacement/(seawaterDensity*maxDraft) + 2.7*lwl*maxDraft)*(1.2-0.34*Cbw)
                                  - 3.0*(maxDraft-actualDraft)*(lwl+breadth)
                                  )#end case 3
                          )#end case 2
                    )#end case 1

  return(wettedSA)
}


rinvgamma <- function(a, b){
  1 / stats::rgamma(1, shape = a, rate = b)
}

dinvgamma <- function (x, shape, scale = 1)  {
  log.density <- shape * log(scale) - lgamma(shape) -
    (shape + 1) * log(x) - (scale /  x)
  return(exp(log.density))
}

update_sigma <- function(a_s, k_s, a_sigma, b_sigma){

  a_s_sites <- a_s[!duplicated(k_s$Site)]
  n <- length(a_s_sites)

  rinvgamma(a_sigma + (n / 2), b_sigma + sum(a_s_sites^2)/2)

}

update_l_sigma_integrated <- function(l_T, sigma_T, a_l_T, b_l_T, a_ig, b_ig, Y, sigma_psi,
                                      z, XbetaY, Xbetas, Xbeta_cov, eps_s, Omega,
                                      b_psi_b, sd_l, sd_sigma_T,  X_y_index, usingSpatial){

  l_T_star <- stats::rnorm(1, l_T, sd_l)
  sigma_T_star <- stats::rnorm(1, sigma_T, sd_sigma_T)

  if(l_T_star < 4 & l_T_star > 0 & sigma_T_star > 0){

    K_l <- K(1:Y, 1:Y, sigma_T^2, l_T) + sigma_psi^2 + diag(exp(-10), nrow = Y)
    K_l_star <- K(1:Y, 1:Y, sigma_T_star^2, l_T_star) + sigma_psi^2 + diag(exp(-10), nrow = Y)

    # term 1

    tXOmegX <- matrixProductXtOmegaX_year(Y, Omega, X_y_index)

    logterm1_1 <- -.5 * log(FastGP::rcppeigen_get_det(K_l_star %*% tXOmegX + diag(1, nrow = Y)))
    logterm1_2 <- -.5 * log(FastGP::rcppeigen_get_det(K_l %*% tXOmegX + diag(1, nrow = Y)))

    logdeterminantsRatio <- logterm1_1 - logterm1_2

    # term 2

    logmuKlmu <- - .5 * b_psi_b %*% (FastGP::rcppeigen_invert_matrix(K_l_star) -
                                       FastGP::rcppeigen_invert_matrix(K_l)) %*% b_psi_b

    # term 3

    if(usingSpatial){
      c_i <- Xbetas + Xbeta_cov + eps_s
    } else {
      c_i <- Xbeta_cov + eps_s
    }

    k_i <- z - .5
    z_tilde <- k_i - c_i * as.vector(Omega)

    Xtz_tilde <- XpsiYz(X_y_index, z_tilde, Y)

    mu_tilde <- Xtz_tilde + FastGP::rcppeigen_invert_matrix(K_l) %*% b_psi_b
    mu_tilde_star <- Xtz_tilde + FastGP::rcppeigen_invert_matrix(K_l_star) %*% b_psi_b

    XtOXpB <- tXOmegX + FastGP::rcppeigen_invert_matrix(K_l)
    XtOXpB_star <- tXOmegX + FastGP::rcppeigen_invert_matrix(K_l_star)

    log_term2ratio_star <- .5 * ( t(mu_tilde_star) %*% FastGP::rcppeigen_invert_matrix(XtOXpB_star) %*% mu_tilde_star )
    log_term2ratio <- .5 * ( t(mu_tilde) %*% FastGP::rcppeigen_invert_matrix(XtOXpB) %*% mu_tilde )

    logterm3 <- log_term2ratio_star - log_term2ratio

    likelihood <- exp(logdeterminantsRatio + logmuKlmu +  logterm3)

    prior_l <- exp(stats::dgamma(l_T_star, a_l_T, b_l_T, log = T) - stats::dgamma(l_T, a_l_T, b_l_T, log = T))
    prior_sigma <- exp(dinvgamma(sigma_T_star, a_ig, b_ig) - dinvgamma(sigma_T, a_ig, b_ig))

    posterior <-  likelihood * prior_l * prior_sigma

    if(!is.na(posterior)){
      if(stats::runif(1) < posterior){
        l_T <- l_T_star
        sigma_T <- sigma_T_star
      }
    }

  }

  list("l_T" = l_T, "sigma_T" = sigma_T)
}

rcpp_log_dmvnorm_fast <- function (inv_S, diag_S, sigma_s, x) {
  n <- length(x)
  # return((-n/2) * log(2 * pi) - sum(log(diag_S)) - (1/2) * x %*% inv_S %*% x)
  return((-n/2) * log(2 * pi) - sum(log(diag_S) + log(sigma_s)) -
           (1/2) * (1 / sigma_s^2) * x %*% inv_S %*% x)
}

sample_l_grid <- function(l_s_grid, inv_K_s_grid, diag_K_s_grid,
                          a_l_s, b_l_s, a_s, sigma_s){

  posterior_val <- rep(NA, length(l_s_grid))

  for (j in 1:length(l_s_grid)) {

    l_s <- l_s_grid[j]
    # K_s_grid_j <- K_s_grid[,,j] * sigma_s^2
    # inv_K_s_grid_j <- inv_K_s_grid[,,j] / sigma_s^2
    # diag_K_s_grid_j <- diag_K_s_grid[,j] * sigma_s

    # loglikelihood <- rcpp_log_dmvnorm_fast(inv_K_s_grid[,,j],
    #                                        diag_K_s_grid[,j], sigma_s, a_s)

    loglikelihood <- ((-length(a_s)/2) * log(2 * pi) - sum(log(diag_K_s_grid[,j]) + log(sigma_s)) -
                        (1/2) * (1 / sigma_s^2) * a_s %*% inv_K_s_grid[,,j] %*% a_s)

    # loglikelihood <- rcpp_log_dmvnorm_fast(1, inv_K_s_grid_j,
    # diag_K_s_grid_j,  a_s)

    # Sigma_l <- K2(X_tilde, X_tilde, sigma_s^2, l_s) + diag(exp(-10), nrow = nrow(X_tilde))

    # (loglikelihood2 <- rcpp_log_dmvnorm( Sigma_l, rep(0, X_centers), a_s, F))

    logPrior <- dgamma(l_s, a_l_s, b_l_s, log = T)

    posterior_val[j] <- logPrior + loglikelihood

  }

  # posterior_val <- sample_l_grid_cpp(l_s_grid, sigma_s, inv_K_s_grid, diag_K_s_grid,
  # a_l_s, b_l_s, a_s)

  posterior_val <- posterior_val - max(posterior_val)

  index_l_grid <- sample(1:length(l_s_grid), size = 1,
                         prob = exp(posterior_val) / sum(exp(posterior_val)))

  index_l_grid
}

update_hyperparameters <- function(l_T, a_l_T, b_l_T, sd_l_T, sd_sigma_T,
                                   sigma_T, a_sigma_T, b_sigma_T, Y,
                                   beta_psi, inv_B_psi,
                                   b_psi, sigma_psi,
                                   l_s_grid, K_s_grid, inv_K_s_grid,
                                   inv_chol_K_s_grid, diag_K_s_grid,
                                   a_l_s, b_l_s,
                                   sigma_s, a_sigma_s, b_sigma_s, X_tilde,
                                   a_sigma_eps, b_sigma_eps,
                                   usingSpatial,
                                   XbetaY, Xbetas, Xbeta_cov,
                                   eps_s, k_s,
                                   z, X_psi, Omega, X_y_index){

  # update l_T and sigma_T -------------------

  list_l_sigma <- update_l_sigma_integrated(l_T, sigma_T, a_l_T, b_l_T, a_sigma_T, b_sigma_T, Y,
                                            sigma_psi,
                                            z, betaY, Xbetas, Xbeta_cov, eps_s, Omega,
                                            b_psi[1:Y], sd_l = sd_l_T,
                                            sd_sigma_T, X_y_index, usingSpatial)
  l_T <- list_l_sigma$l_T
  sigma_T <- list_l_sigma$sigma_T

  # reupdate inv_B_psi ------------

  K_l <- K(1:Y, 1:Y, sigma_T^2, l_T) + sigma_psi^2 + diag(exp(-8), nrow = Y)

  inverse_Kl <- FastGP::rcppeigen_invert_matrix(K_l)
  inverse_Kl[lower.tri(K_l)] <- t(inverse_Kl)[lower.tri(K_l)]

  inv_B_psi[1:Y, 1:Y] <- inverse_Kl

  # update l_s -------------------

  if(usingSpatial){
    index_ls_grid <- sample_l_grid(l_s_grid, inv_K_s_grid, diag_K_s_grid,
                                   a_l_s, b_l_s, beta_psi[Y + 1:nrow(X_tilde)], sigma_s)
    l_s <- l_s_grid[index_ls_grid]
  } else {
    l_s <- 0
    index_ls_grid <- 0
  }

  # update sigma_s ---------------------

  if(usingSpatial){

    X_centers <- nrow(X_tilde)

    inv_chol_Kl <- inv_chol_K_s_grid[,,index_ls_grid]

    a_s <- beta_psi[Y + 1:X_centers]

    a_ig <- X_centers / 2
    Ltas <- inv_chol_Kl %*% a_s
    b_ig <- (t(Ltas) %*% Ltas) / 2

    sigma_s <- sqrt(rinvgamma(a_sigma_s + a_ig, b_sigma_s + b_ig))
  }

  # reupdate inv_B_psi ------------

  if(usingSpatial){
    X_centers <- nrow(X_tilde)

    inv_K_lsigma <- inv_K_s_grid[,,index_ls_grid] / sigma_s^2

    inv_B_psi[Y + 1:X_centers, Y + 1:X_centers] <- inv_K_lsigma
  }

  # update sigma_as ------------------------------

  sigma_eps <- sqrt(update_sigma(eps_s, k_s, a_sigma_eps, b_sigma_eps))

  list("l_T" = l_T,
       "sigma_T" = sigma_T,
       "inv_B_psi" = inv_B_psi,
       "l_s" = l_s,
       "index_ls_grid" = index_ls_grid,
       "sigma_s" = sigma_s,
       "sigma_eps" = sigma_eps)
}


#http://esapubs.org/archive/ecol/E095/262/STARS.R

# Modified from Sergei Rodionov's VBA and Andrew Gardner's Matlab codes to produce an R version of STARS. Go to http://www.climatelogic.com/overview for more information, or see the Rodionov (2004; 2006) references in the paper.

# Red noise correction included, but some other functions from VBA code (version 3.2) still missing (e.g. tests in shifts in variance).

# Checked RSI results using two independent time-series against Rodionov plug in.

# Alistair Seddon, October 2013.
# alistair.seddon@gmail.com

stars<- function(y=c(rnorm(50), rnorm(50,2,1)), L=20, p=0.05, h=1,  AR1red="none", prewhitening = F) {

  if(AR1red=="none" && prewhitening== T) stop("Impossible to perform prewhitening if AR1 coefficient is not estimated")

  m= round((L+1)/3)				# formula to estimate subsample size for calculating alpha (Rodionov 2006 + http://www.climatelogic.com/documentation/red-noise-estimation)

  # library("MASS") needed if you want to use Huber correction parameter built in to MASS

  # -------------------------------------------------------------------------
  #    hWeightedAverage(xwin)

  #     Calculates the mean estimate for a given range using Huber's weights.
  # -------------------------------------------------------------------------

  hWeightedAverage<-function(xwin, h){

    # simple estimate of the regime mean for the windowed clip
    dblEstAve <- mean(xwin);

    for(jjj in 1:2){
      sumWeights = 0
      sumAve = 0

      # Estimate normalised deviation
      xDev = (xwin-dblEstAve)/sqrt(sigL)

      # Estimate weights or normalised deviation
      xDev[xDev==0] = 1
      wDev = pmin(rep(1, length(xwin)), h/abs(xDev), na.rm=T)

      #sum weights and weighed values
      sumWeights = sum(wDev)
      sumAve = sum(xDev*wDev)

      sumAve = sumAve/sumWeights
      sumAve = sumAve*sqrt(sigL) + dblEstAve
      dblEstAve = sumAve
    }

    dblWeightedAve = dblEstAve
    # hestimate<- huber(xwin, h)
    # dblWeightedAve = hestimate$mu
  }

  #-------------------------------------------------------------------
  # estimateSigma
  # Estimate the long-term, L-pt variance (assume homoskedastic signals).
  #-------------------------------------------------------------------

  estimateSigma<-function(x, L){

    # Estimate the long-term length-L variance. If the signal >> length of the analysis window, sample to estimate the variance.

    nx<-length(x)
    if(nx/L>300) ix <- as.integer(runif(100)*(nx-2*L)+L) else ix<-seq(L,nx,1)
    s<-0
    for(i in 1:length(ix)){
      xwin <- x[(ix[i]-L+1):ix[i]]
      s <- s + var(xwin, na.rm=T)
    }
    sigL1 = s / length(ix)
    sigL1
  }

  # ------------------------------------------------------------------
  #   getThreshold()
  #
  #   Calculate the critical threshold of deviation that signals regime changes.  This does not change over the signal.
  # ---------------------------------------------------------------
  getThreshold<-function(L, p, sigL){

    if(prewhitening == T){
      dof <- 2*L-2						# number degrees freedom
    } else {
      dof <- EqN((2*L-2), alpha)
    }

    t <- abs(qt(p/2, dof));              # crit 2-sided t-value
    thresh1 = t*sqrt(2*sigL/L);          # crit deviation
    thresh1
  }

  # -------------------------------------------------------------------

  # OLS estimate of AR1 coefficient from Orcutt and Winokur, 1969, Econometrica, 37:1,1-14

  # -------------------------------------------------------------------

  OLScalc<-function(x){
    Nobs = length(x)

    ave1 = mean(x[2:Nobs])
    ave2 = mean(x[1:(Nobs-1)])

    sumNom=0
    sumDenom=0
    for(i in 2:Nobs){
      sumNom = sumNom + (x[i] - ave1) * (x[i - 1] - ave2)
      sumDenom = sumDenom + (x[i - 1] - ave2) * (x[i - 1] - ave2)
    }
    if(sumDenom > 0) OLSAR1 = sumNom / sumDenom else OLSAR1 = 0
    OLSAR1
  }

  # -------------------------------------------------------------------
  # AR1 correlation estimate (alpha)

  # -------------------------------------------------------------------

  AR1cor<-function (m, y){
    m= m #define this in big function above

    ny=length(y)
    iy=seq(1,ny-m+1,1)
    OLS=rep(NA, length(iy))

    # Calculate OLS for sequential samples of length m
    for(i in 1:length(iy)){

      xwin = y[(iy[i]):(iy[i]+m-1)]

      if(length(xwin[is.na(xwin)]) == 0)   OLS[i] <- OLScalc(xwin)
    }

    est<-median(OLS, na.rm=T)

    # Calculate IP4
    IP4= est + 1/m
    for(j in 1:3) IP4=IP4 + abs(IP4)/m

    # Calculate MPK
    if (m>4) MPK=((m-1)*est+1)/(m-4) else MPK= est

    alphaEst<-c(est, MPK, IP4)

  }

  # -------------------------------------------------------------------
  # Function EqP: calculates t-test using equivalent sample size as in von Storch and Zwiers (1999, p.115)
  # -------------------------------------------------------------------

  EqP= function(rng1, rng2){

    # Set standard no-result for if command at end
    EqP = 0

    # Calculate means and variances
    ave1 = mean(rng1, na.rm =T)
    ave2 = mean(rng2, na.rm =T)

    var1 = sd(rng1, na.rm = T)
    var2 = sd(rng2, na.rm = T)

    # Calculate effective sample sizes
    Ns1 = length(na.omit(rng1))
    if(Ns1 < 2){
      EqP = -1
    }
    eN1 = EqN(Ns1, alpha)

    Ns2 = length(na.omit(rng2))
    if(Ns2 < 2){
      EqP = -1
    }
    eN2 = EqN(Ns2, alpha)

    if(EqP == -1){
      EqP
    } else{
      # Calculate t-statistics
      T_stat = sqrt(var1/eN1 + var2/ eN2)
      T_stat = abs(ave1 - ave2)/ T_stat

      EqP = (1-pt(T_stat, eN1 + eN2 -2))*2
      EqP
    }

  }

  # -------------------------------------------------------------------
  # EqN: Calculates equivalent sample size as in von Storch and Zwiers (1999, p.115)
  # -------------------------------------------------------------------

  EqN = function(Ns, alpha){

    sumEqN = rep(NA, Ns-1)
    for(i in 1: (Ns-1)){
      sumEqN[i] = (1-i/Ns)*alpha^i
    }

    EqN = Ns / (1 + sum(c(sumEqN)))

    # just in case
    if( EqN <=2) EqN = 2
    if(EqN > Ns) EqN =Ns
    EqN
  }

  # ------------------------------------------------------------------
  #   cusumUp()

  #       Compute the L-pt cusum for positive level changes.  For a positive regime change to be accepted, we require the L-pt lookahead samples	to produce a cusum sequence which does not go negative.

  # -------------------------------------------------------------------

  cusumUp<-function(k){
    # LL sets the look ahead length: L, or the number of points until the end of the signal. k is the sample point running through the iteration
    LL <- min(L, N-k+1)

    # dblXdev is the length-LL vector of normalized deviations of x outside of the range lvl +/- thresh
    dblXdev = ((x[k:(k+LL-1)]) - (lvl+thresh)) / sqrt(sigL)

    #  these are Huber weight values, so large deviations are deemphasized
    dblXdev[dblXdev==0] = 1
    dblXweight = pmin(rep(1, length(dblXdev)), h/abs(dblXdev), na.rm=T)

    # % the cusum is the integral of the weighted deviations; we normalize
    # % here, too, by dividing by the sum of the weights

    cs<- cumsum(dblXweight*dblXdev)/sum(dblXweight)

    # cs<-cumsum(dblXdev) #simple non weighted version

    # we check for cusum values below zero, which would indicate a failed
    # regime change; otherwise, we have a positive shift
    if (length(which(cs < 0) > 0)) cs = 0 else  cs = cs[LL]
    cs
  }

  # ------------------------------------------------------------------
  #   cusumDown()

  #       Compute the L-pt cusum for positive level changes.  For a positive regime change to be accepted, we require the L-pt lookahead samples	to produce a cusum sequence which does not go negative.

  # -------------------------------------------------------------------

  cusumDown<-function(k){
    # LL sets the look ahead length: L, or the number of points until the end of the signal. k is the sample point running through the iteration
    LL <- min(L, N-k+1)

    # dblXdev is the length-LL vector of normalized deviations of x outside of the range lvl +/- thresh
    dblXdev = ((x[k:(k+LL-1)]) - (lvl-thresh)) / sqrt(sigL)

    #  these are Huber weight values, so large deviations are deemphasized
    dblXdev[dblXdev==0] = 1
    dblXweight = pmin(rep(1, length(dblXdev)), h/abs(dblXdev), na.rm=T)

    # % the cusum is the integral of the weighted deviations; we normalize
    # % here, too, by dividing by the sum of the weights

    cs<- cumsum(dblXweight*dblXdev)/sum(dblXweight)

    # cs<-cumsum(dblXdev) # simple non-weighted version

    # we check for cusum values above zero, which would indicate a failed
    # regime change; otherwise, we have a positive shift
    if (length(which(cs > 0) > 0)) cs = 0 else  cs = cs[LL]
    cs

  }

  #  -------------------------------------------------------------------------
  #      rsi(k)

  #      Compute the rsi for a given sample index, regime mean, and critical
  #      threshold.
  #    -------------------------------------------------------------------------
  rsi<-function(k){
    if(x[k] > (lvl + thresh)){
      r = cusumUp(k)
    } else if(x[k] < (lvl - thresh)){
      r = cusumDown(k)
    } else {
      r = 0
    }
    r
  }

  #  -------------------------------------------------------------------------

  # Red noise filtering of timeseries.

  #  -------------------------------------------------------------------------

  alpha<-AR1cor(m,y) # calculate alpha estimates

  if(AR1red=="est"){
    alpha = alpha[1]
  }else  if(AR1red=="MPK"){
    alpha = alpha[2]
  }else if(AR1red=="IP4"){
    alpha = alpha[3]
  }else if(AR1red=="none"){
    alpha= 0
  }

  if(alpha<0) alpha <- 0 ; if(alpha>1) alpha <- 1

  # Filter time series if selected and select as x for main procedure, otherwise use timeseries

  if(prewhitening == T){
    Zt=rep(NA, length(y))
    for(j in 2:length(y)) Zt[j]<-y[j]-(y[j-1]*alpha)

    if(alpha>0) x=Zt[-1] else x=y[-1]
    names(x) <- names(y)[-1]

  } else x=y[-1]

  x <- na.omit(x)

  #  -------------------------------------------------------------------------

  # initialisation

  #  -------------------------------------------------------------------------

  sigL = estimateSigma(x, L);           			# sample L-pt variance
  thresh = getThreshold(L, p, sigL);          # critical threshold
  lvl = hWeightedAverage(x[1:L], h);             # initial mean level
  R = rep(0, length(x));                      # rsi values
  RpVal<-rep(0, length(x))
  cp = 1;                                     # current change-point index
  N = length(x)                              # number of samples

  if(length(names(y))==0) {
    stop("Stopped: No ages supplied with timeseries")

  } else ages = names(y)

  # Main routine.
  for (k in 2:N){
    R[k] = rsi(k)

    #   too few samples to confirm last regime change?
    if (abs(R[k]) > 0 && k > (N-L+1)) break

    #   test for regime shifts and update current regime mean (unless we are within L-pts of most recent change-point)
    if(R[k] == 0){
      if(k >= (cp + L)) lvl = hWeightedAverage(x[cp:k], h)    # same regime, far enough

    } else{
      cp = k                              # regime change
      lvl = hWeightedAverage(x[k:(k+L-1)], h); # same regime, far enough from cp
    }
  }

  #  Calculation of new regime sample means and associated pvalues of shifts)
  if(R[length(R)] != 0) R[length(R)]<- 0
  cps<-which(abs(R)>0)

  rID<-rep(1, length(x))
  rLabel<-seq(2, length(cps)+1,1)
  Rmean<-rep(0, length(cps)+1)

  for(j in 1:length(cps)) rID[cps[j]:N]<-rLabel[j]
  for(j in 1:length(Rmean)) Rmean[j]<- hWeightedAverage(x[rID==j], h)
  # for(j in 1:length(Rmean)) Rmean[j]<- mean(x[rID==j])
  xNames= names(x)

  rID1 = rID
  for(j in 1:length(Rmean)) rID[rID==j]<-Rmean[j]

  xNA=rep(NA, length(y))
  xNA[match(xNames, names(y))] <- x

  RNA=rep(NA, length(y))
  RNA[match(xNames, names(y))] <- c(R)

  rIDNA=rep(NA, length(y))
  rIDNA[match(xNames, names(y))] <- c(rID)
  starsResult<-cbind(y,xNA, RNA , rIDNA)

  colnames(starsResult) = c("ts", "AR1.cor", "rsi", "mean"); rownames(starsResult) = ages

  # Estimate pValues of shifts on either white-noise filtered series, or by using the AR1 correction parameter

  pVal = rep(0, length(cps))

  for(j in 1:length(cps)) {

    rs1 = x[rID1==j]
    rs2 = x[rID1==(j+1)]

    if(length(rs2)==1) {
      next
    } else {
      ifelse(prewhitening ==T, pVal[j] <- t.test(rs1, rs2)$p.value, pVal[j] <- EqP(rs1, rs2))
    }
  }

  if(length(which(pVal == -1)) >0) warning("pValue calculation of -1 due regime containing only one sample")

  starsOUT=list(starsResult, alpha, pVal)
  names(starsOUT)=c("starsResult", "alpha", "pVal")
  starsOUT

}


##################################################
##### Manipulate raster and netCDF datasets ######
##### J Chave 15 Dec 2013                   ######
##### M Rejou-Mechain & J Chave 25 Mar 2014 ######
##### Written in R version 3.0.3            ######
##################################################

#=========================================================================
## In Chave et al. E and CWD are environmental variables
## CWD is long-term climatic water deficit (see Chave et al for a precise definition)
## TS is temperature seasonality extracted from the Worldclim dataset
## PS is the precipitation seasonality extracted from the Worldclim dataset
## E is an environmental stress variable defined in Eq (6b) of Chave et al as:
## E=(0,178*TS-0.938*CWD-6.61*PS)/1000
##
## The layers are provided at 2.5 arc-minute resolution, or ca. 5 km
## This 24 cells per degree, or 0.041666666666667 degree per cell
# For 'raster' beginners:
# raster starts at top left corner; Latitude is from +90 to -60, longitude from -180 to +180
# thus, French Guiana is at (+5,-55), or 85 rows from top and (180-55)=125 cols from left.
# grain size is 0.041666666666667 degree, hence there are 24 cells per degree, or about 1 cell every 5 km
#=========================================================================

## filename is either CWD or E
## format 'nc' denotes a file in netCDF format  (default; much faster in linux-based environments)
## format 'bil' denotes a file in raster format
## default interpolation method used here is 'bilinear', where the exact value at the coordinates
## is interpolated along both the x and y axes. The other option is 'simple' (the cell value is retrieved)

retrieve_raster=function(filename,coord,plot=F,format="nc"){
  require("raster")
  if(format=='nc') require("ncdf")
  zipurl <- "http://chave.ups-tlse.fr/pantropical_allometry"
  if(format=='nc') zipurl = paste(zipurl,"/",filename,".nc.zip",sep="")
  if(format=='bil') zipurl = paste(zipurl,"/",filename,".bil.zip",sep="")
  # Read the raster file in netCDF format from
  # http://chave.ups-tlse.fr/pantropical_allometry.htm
  print(zipurl);
  DEMzip <- download.file(zipurl, destfile = "zipdir")
  unzip("zipdir", exdir = "unzipdir")
  nam=paste("unzipdir/",filename,".nc",sep="")
  RAST <- raster(nam)
  # Check that the dataset has properly been imported and that
  # your coordinates are correct
  # This step takes time
  if(plot==T){
    plot(RAST)
    points(coord,pch="x")
  }
  # Extract the raster value
  # coord=cbind(longitude,latitude)
  RASTval=extract(RAST,coord,method="bilinear")
  return(RASTval)
}


#' Global Distance Metric Learning
#'
#' Performs Global Distance Metric Learning (GDM) on the given data, learning a full matrix.
#'
#' Put GdmFull function details here.
#'
#' @param data \code{n * d} data matrix. \code{n} is the number of data points,
#'             \code{d} is the dimension of the data.
#'             Each data point is a row in the matrix.
#' @param simi \code{n * 2} matrix describing the similar constrains.
#'              Each row of matrix is serial number of a similar pair in the original data.
#'				For example, pair(1, 3) represents the first observation is similar the 3th observation in the original data.
#' @param dism \code{n * 2} matrix describing the dissimilar constrains as \code{simi}.
#'				Each row of matrix is serial number of a dissimilar pair in the original data.
#' @param maxiter numeric, the number of iteration.
#'
#' @return list of the GdmDiag results:
#' \item{newData}{GdmDiag transformed data}
#' \item{fullA}{suggested Mahalanobis matrix}
#' \item{dmlA}{matrix to transform data, square root of diagonalA }
#' \item{converged}{whether the iteration-projection optimization is converged or not}
#'
#' For every two original data points (x1, x2) in newData (y1, y2):
#'
#' \eqn{(x2 - x1)' * A * (x2 - x1) = || (x2 - x1) * B ||^2 = || y2 - y1 ||^2}
#'
#' @keywords GDM global distance metirc learning transformation mahalanobis metric
#'
#' @note Be sure to check whether the dimension of original data and constrains' format are valid for the function.
#'
#' @author Tao Gao <\url{http://www.gaotao.name}>
#'
#' @references
#' Steven C.H. Hoi, W. Liu, M.R. Lyu and W.Y. Ma (2003).
#' Distance metric learning, with application to clustering with side-information.
#  in \emph{Proc. NIPS}.
#'

#' @examples
#' set.seed(123)
#' library(MASS)
#' library(scatterplot3d)
#'
#' # generate simulated Gaussian data
#' k = 100
#' m <- matrix(c(1, 0.5, 1, 0.5, 2, -1, 1, -1, 3), nrow =3, byrow = T)
#' x1 <- mvrnorm(k, mu = c(1, 1, 1), Sigma = m)
#' x2 <- mvrnorm(k, mu = c(-1, 0, 0), Sigma = m)
#' data <- rbind(x1, x2)
#'
#' # define similar constrains
#' simi <- rbind(t(combn(1:k, 2)), t(combn((k+1):(2*k), 2)))
#'
#' temp <-  as.data.frame(t(simi))
#' tol <- as.data.frame(combn(1:(2*k), 2))
#'
#' # define disimilar constrains
#' dism <- t(as.matrix(tol[!tol %in% simi]))
#'
#' # transform data using GdmFull
#' result <- GdmFull(data, simi, dism)
#' newData <- result$newData
#' # plot original data
#' color <- gl(2, k, labels = c("red", "blue"))
#' par(mfrow = c(2, 1), mar = rep(0, 4) + 0.1)
#' scatterplot3d(data, color = color, cex.symbols = 0.6,
#'			  xlim = range(data[, 1], newData[, 1]),
#'			  ylim = range(data[, 2], newData[, 2]),
#'			  zlim = range(data[, 3], newData[, 3]),
#'			  main = "Original Data")
#' # plot GdmFull transformed data
#' scatterplot3d(newData, color = color, cex.symbols = 0.6,
#'			  xlim = range(data[, 1], newData[, 1]),
#'			  ylim = range(data[, 2], newData[, 2]),
#'			  zlim = range(data[, 3], newData[, 3]),
#'			  main = "Transformed Data")

GdmFull <- function(data, simi, dism, maxiter = 100) {
		data <- as.matrix(data)
		N <- dim(data)[1]
		d <- dim(data)[2]
		new.simi <- unique(t(apply(simi, 1, sort)))
		new.dism <- unique(t(apply(dism, 1, sort)))

		A <- diag(1, d) * 0.1
		W <- mat.or.vec(d, d)
		dij <- mat.or.vec(1, d)

		# sphereMult = cov(data)^(-0.5);
		# spheredata = data %*% sphereMult

		dist1.simi <- data[new.simi[, 1], ] - data[new.simi[, 2], ]
		dist2.ij <- t(apply(dist1.simi, 1, function(x) outer(x, x)))
		W <- matrix(apply(dist2.ij, 2, sum), ncol = d, byrow = TRUE)

		w <- matrix(W, ncol = 1)
		t0 <- as.numeric(crossprod(w, matrix(A, ncol = 1))/100)

		IterProjection <- function(data, simi, dism, A, w, t0 , maxiter = 100) {
							data <- as.matrix(data)
							N = dim(data)[1]     # number of examples
							d = dim(data)[2]     # dimensionality of examples
							# S1 <- mat.or.vec(N, N)
							# D1 <- mat.or.vec(N, N)
							# simi <- rbind(simi, simi[, c(2, 1)])
							# dism <- rbind(dism, dism[, c(2, 1)])
							# S1[simi] <- 1
							# D1[dism] <- 1
							new.simi <- unique(t(apply(simi, 1, sort)))
							new.dism <- unique(t(apply(dism, 1, sort)))

							# error1=1e5
							threshold2 <- 0.01  # error-bound of main A-update iteration
							epsilon <- 0.01   # error-bound of iterative projection on C1 and C2
							maxcount <- 200

							w1 <- w/norm(w, "F")    # make 'w' a unit vector
							t1 <- t0/norm(w, "F")

							count <- 1
							alpha <- 0.1    # initial step size along gradient

							GradProjection <- function(grad1, grad2, d) {
												g1 <- matrix(grad1, ncol = 1)
												g2 <- matrix(grad2, ncol = 1)

												g2 <- g2/norm(g2, "F")
												gtemp <- g1 - as.numeric(crossprod(g2, g1)) * g2
												gtemp <- gtemp/norm(gtemp, "F")
												grad.proj <- matrix(gtemp, d, d)
												return(grad.proj)
							}

							fS1 <- function(data, new.simi, A, N, d, fudge = 0.000001) {

									dist1.simi <- data[new.simi[, 1], ] - data[new.simi[, 2], ]
									dist2.ij <- t(apply(dist1.simi, 1, function(x) outer(x, x)))
									fs.1d <- matrix(apply(dist2.ij, 2, sum), ncol = d, byrow = TRUE)
									return(fs.1d)
							}

							fD1 <- function(data, new.simi, A, N, d, fudge = 0.000001) {
									dist1.dism <- data[new.dism[, 1], ] - data[new.dism[, 2], ]
									dist.ij <- numeric(dim(dist1.dism)[1])
									for (i in 1:dim(dist1.dism)[1]) {
										dist.ij[i] <- sqrt(t(dist1.dism[i, ]) %*% A %*% t(t(dist1.dism[i, ])))
									}
									sum.dist <- sum(dist.ij) + 0.000001
									Mij <- t(apply(dist1.dism, 1, function(x) outer(x, x)))
									temp <- cbind(Mij, t(t(dist.ij)))

									deri.ij <- 0.5 * temp[, 1:(d^2)]/(temp[, d^2 + 1] + (temp[, d^2 + 1] == 0) * fudge)
									sum.deri <- matrix(apply(deri.ij, 2, sum), ncol = d, byrow = TRUE)
									fd.1d <- sum.deri/sum.dist
									return(fd.1d)
							}

							fD <- function(data, new.dism, A, N, d) {
									dist1.dism <- data[new.dism[, 1], ] - data[new.dism[, 2], ]
									dist.ij <- numeric(dim(dist1.dism)[1])
									for (i in 1:dim(dist1.dism)[1]) {
										dist.ij[i] <- sqrt(t(dist1.dism[i, ]) %*% A %*% t(t(dist1.dism[i, ])))
									}
									fd <- sum(dist.ij) + 0.000001
									fd <- log(fd)
									return(fd)
							}

							grad1 <- fS1(data, new.simi, A, N, d);   # gradient of similarity constraint function
							grad2 <- fD1(data, new.dism, A, N, d);   # gradient of dissimilarity constraint func.
							M <- GradProjection(grad1, grad2, d); # gradient of fD1 orthognal to fS1

							A.last <- A        # initial A
							done <- 0
							delta <- 0
							converged <- 0
							while (done == 0) {
									projection.iters <- 0
									satisfy <- 0

									while (projection.iters < maxiter & satisfy == 0) {
										A0 <- A
										x0 <- matrix(A0, ncol = 1)
										if(crossprod(w, x0) <= t0)
											A <- A0
										else {
											x <- x0 + as.numeric(t1 - crossprod(w1, x0)) * w1
											A <- matrix(x, 3, 3)
										}

										A <- (A + t(A))/2
										vl <- eigen(A)
										vl[[1]][vl[[1]] < 0] = 0
										A <- vl[[2]] %*% diag(vl[[1]], d) %*% t(vl[[2]])

										fDC2 <- crossprod(w, matrix(A, ncol = 1))
										error1 <- as.numeric((fDC2 - t0)/t0)
										projection.iters <- projection.iters + 1
										satisfy <- as.numeric(ifelse(error1 > epsilon, 0, 1))
									}

									obj.previous <- fD(data, new.dism, A.last, N, d)
									obj <- fD(data, new.dism, A, N, d)

									if (obj > obj.previous & satisfy == 1) {
										alpha <-  alpha * 1.05
										A.last <- A
										grad2 <- fS1(data, new.simi, A, N, d)
										grad1 <- fD1(data, new.dism, A, N, d)
										M <- GradProjection(grad1, grad2, d)
										A <- A + alpha * M
									}
									else{
										alpha <- alpha/2
										A <- A.last + alpha * M
									}
									delta <- norm(alpha * M, "F")/norm(A.last, "F")
									count <- count + 1
									done <- ifelse(delta < threshold2 | count == maxcount, 1, 0)
							}
							converged <- ifelse(delta > threshold2, 0, 1)
							return(list("converged" = ifelse(converged == 1, "Yes", "No"), "fullA" = A))
		}

		iterproj <- IterProjection(data, simi, dism, A, w, t0)
		eigenvalue <- eigen(iterproj$fullA)
		dml <- eigenvalue[[2]] %*% sqrt(diag(eigenvalue[[1]], d))
		newData <- data %*% dml
		return(list("newData" = newData, "fullA" = iterproj[[2]], "dmlA" = dml, "converged" = iterproj[[1]]))
}


# Synthetic Data Project
# Common Functions

# Author:
# Version:  5/14/2017

# Copyright 2017

# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
# associated documentation files (the "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the
# following conditions:

# The above copyright notice and this permission notice shall be included in all copies or substantial
# portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
# LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

frequencyMode <- function(x, nbins=25, nTopMean=1) {

  # compute mode of vector x as weighted mean of top nTopMean frequencies using nbins cut intervals
  # tabulate frequencies
  # take weighted mean of all bin midpoints in top nTopMean frequencies
  xtab <- table(cut(x, nbins))
  # identify top nTopMean frequencies
  k <- which(xtab %in% sort(unique(xtab), decreasing=T)[1:nTopMean])
  sum(mapply(k, FUN=function(i) {
                      # multiply interval centerpoints by frequencies
                      # format is "(b0,b1]" from cut()
                      interval <- names(xtab)[i]
                      # locate comma
                      cpos <- regexpr(",", interval)[1]
                      # exclude leading ( and trailing ]
                      b0 <- as.numeric(substr(interval, 2, cpos-1))
                      b1 <- as.numeric(substr(interval, cpos+1, nchar(interval)-1))
                      # multiply frequency by midpoint of lower and upper bounds
                      as.numeric(xtab[i])*(b0+b1)/2
                    })) / sum(xtab[k])

}


# ########################################
# ########## FUNCTIONS SECTION ############

###### CREATE BASIC ADJACENCY MATRIX #####
# no edge weights, useful for DCI with no distance decay
# rbgl requires adjacency matrix
# FIPEX outputs adjacency table so need to convert
# note: igraph requires only adjacency table

# library(rbenchmark) can be used to test
#https://stackoverflow.com/questions/34355892/build-a-square-adjacency-matrix-from-data-frame-or-data-tablelevs <- unique(unlist(neighbournodes_all, use.names=F))

create_basic_adjmatrix_2020 <- function(neighbournodes_all,option="2020_tapply") {
    # binary adjacency matrix defining connectivity

    if(option=="2020_tapply"){
        neighbournodes_all$edgeLength <- 1
        adj_matrix <- with(neighbournodes_all, tapply(edgeLength, list(Node1, Node2),FUN=length, default = 0))

    }else if(option=="2020_dfmatrix"){
        matrix_table <- table(neighbournodes_all)
        adj_mat1 <- as.data.frame.matrix(matrix_table)
        adj_matrix <- as.matrix(adj_mat1)

    }else if(option=="oldway"){

        # pre-2020 code
        segments<-with(neighbournodes_all, unique(Node2))
        #create a matrix with only 0's in them
        adj_matrix<-matrix(nrow=length(segments),
                       ncol=length(segments),
                       rep(0,length(segments)*length(segments)))
        segment_length<-length(segments)
        rownames(adj_matrix)<-colnames(adj_matrix)<-segments

        for (i in 1:segment_length){
            # find the segments in segment.matrix$Seg where segment.matrix$Seg_ID matches segments[i]
            # index of matching positions - this will be a vector of 1's and NA's
            pos_match<-	match(neighbournodes_all$Node2,segments[i])

            # keep only the positions where pos.match==1
            adj_segments<-neighbournodes_all$Node1[!is.na(pos_match)]

            # find the column positions that correspond to the adjacaent segments
            col<-match(adj_segments,segments)

            # the row number should correspond to i
            row<-i

            # assign a value of 1 for all values of row and col
            adj_matrix[row,col]<-1
        }
    }
    adj_matrix
}

################################################################
# edge-weighted connectivity list (edges_all) and matrix - for DCI w/ DD
# nodes / edges are not reversed as in original way
# do not need self-connected

create_advanced_adjmatrix_2020 <- function(FIPEX_table=NULL){

    # get connectivity list, convert columns to characters
    FIPEX_table_DD <- FIPEX_table %>% select(NodeEID,DownstreamEID,DownstreamNeighDistance) %>%
    mutate(temp = as.character(NodeEID)) %>%
    mutate(NodeEID = temp) %>%
    mutate(temp = as.character(DownstreamEID)) %>%
    mutate(DownstreamEID = temp) %>%
    select(-temp) %>%
    mutate(DownstreamEID = ifelse(DownstreamEID == "Sink","sink",DownstreamEID))

    # downstream neighbours
    edges_down <- FIPEX_table_DD

    # upstream neigbours
    edges_up <- FIPEX_table_DD %>% select(NodeEID,DownstreamEID,DownstreamNeighDistance) %>%
    mutate(temp = DownstreamEID) %>%
    mutate(DownstreamEID = NodeEID) %>%
    mutate(NodeEID = temp) %>%
    select(-temp)

    # self-connected
    #FIPEX_table %>% select(NodeEID,DownstreamEID,DownstreamNeighDistance) %>%
    #mutate(NodeEID = DownstreamEID, DownstreamNeighDistance = 0.1)

    # merge
    edges_all <- edges_down %>%
    bind_rows(edges_up) %>%
    #add_row(NodeEID = "sink", DownstreamEID = "sink") %>%
    rename(Node1 = NodeEID, Node2 = DownstreamEID, edgeLength = DownstreamNeighDistance)

    # convert to matrix
    adj_matrix_edgelengths <- with(edges_all, tapply(edgeLength, list(Node1, Node2), FUN=sum, default = 0))

    return(adj_matrix_edgelengths)
}

################################################################
# CREATE GRAPH OBJECT for DCI w/ Distance Decay
# edge weights are used for distance calculations
# edge data / attributes used for habitat quantity calculations
# #https://www.rdocumentation.org/packages/graph/versions/1.50.0/topics/graphAM-class
create_graph_dd_2020 <- function(adj_matrix_edgelengths=0.0,FIPEX_table=NULL){

    # Create graph object
    # 2020 - different way to call the graphAM function
    # vs pre-2020
    g_dd <- graphAM(adjMat=adj_matrix_edgelengths,  edgemode="directed", values=list(weight=1))

    # associate passabilities with nodes using NodeData slot
    # e.g. nodeData(g,n=c("b", "c"), attr ="color") <- "red"
    nodeDataDefaults(g_dd, attr ="pass") <- 1.0
    nodeData(g_dd,n=as.character(FIPEX_table$NodeEID), attr="pass") <- as.double(FIPEX_table$BarrierPerm)
    #nd <- nodes(g_dd)

    nodeDataDefaults(g_dd, attr ="nodelabel") <- "none"
    nodeData(g_dd,n=as.character(FIPEX_table$NodeEID), attr="nodelabel") <- as.character(FIPEX_table$NodeLabel)
    nodeData(g_dd,n="sink", attr="nodelabel") <- "sink"
    #nd <- nodes(g_dd)

    nodeDataDefaults(g_dd, attr ="downnodelabel") <- "none"
    nodeData(g_dd,n=as.character(FIPEX_table$NodeEID), attr="downnodelabel") <- as.character(FIPEX_table$DownstreamNodeLabel)
    #nd <- nodes(g_dd)

    nodeDataDefaults(g_dd, attr ="natural") <- "none"
    nodeData(g_dd,n=as.character(FIPEX_table$NodeEID), attr="natural") <- FIPEX_table$NaturalTF
    nodeData(g_dd,n="sink", attr="natural") <- FALSE
    #nd <- nodes(g_dd)

    # optionally can give edges attributes
    #edgeDataDefaults(g_dd, attr="name")<-"noname"
    #edgeData(self, from, to, attr)
    #edgeData(self, from, to, attr) <- value
    edgeDataDefaults(g_dd, attr="HabitatQuan")<-0.0
    edgeData(g_dd,from=as.character(FIPEX_table$NodeEID),
         to=as.character(FIPEX_table$DownstreamEID),
         attr="HabitatQuan")<-as.double(FIPEX_table$HabQuantity)
    # reverse - attr associated with each direction along one edge
    edgeData(g_dd,from=as.character(FIPEX_table$DownstreamEID),
         to=as.character(FIPEX_table$NodeEID),
         attr="HabitatQuan")<-as.double(FIPEX_table$HabQuantity)

    # give edges an easy-to-access name insensitive to direction
    # this is done to quickly identify duplicates later
    # there may be alternatives such as accessing edgeNames but I suspect
    # they are slower than this
    edgeDataDefaults(g_dd, attr="EdgeNameGO")<-"init"
    edgeData(g_dd,from=as.character(FIPEX_table$NodeEID),
         to=as.character(FIPEX_table$DownstreamEID),
         attr="EdgeNameGO")<-paste(as.character(FIPEX_table$DownstreamEID),
                                   as.character(FIPEX_table$NodeEID),
                                   sep="-")
    # reverse - attr associated with each direction along one edge
    edgeData(g_dd,from=as.character(FIPEX_table$DownstreamEID),
         to=as.character(FIPEX_table$NodeEID),
         attr="EdgeNameGO")<-paste(as.character(FIPEX_table$DownstreamEID),
                                   as.character(FIPEX_table$NodeEID),
                                   sep="-")
    return(g_dd)
}

################################################################################
# get all distances and paths (from penult) to all nodes from Sink / all nodes
# https://www.rdocumentation.org/packages/RBGL/versions/1.48.1/topics/dijkstra.sp
# note it must be node-node - no edge-edge possible
# note using this function repeatedly during DCIp is inefficient
# !!! (should use BFS w/ LCA i.e., custom algorithm) !!!
#   (cannot edit the source for Djikstra.sp because it's actually an
#   interface to C++ 'Boost'library for graphs - can't get edge and node attributes
#   during net traversal)

get_paths_distances <- function(g=NULL,fromnode="sink"){
    dijkstra.sp(g,fromnode,eW=unlist(edgeWeights(g)))

    # TO DO: ALTERNATIVES FOR BENCHMARKING
}

##############################################################################
##### SUMMARY TABLE 2020 #####

# replaces similar pre-2020 function to create a table for each edge-edge pair
# Includes options for alternative data management for benchmarking
# (code could be trimmed).
# this function could be sped up with custom algorithm that can find path
# while also grabbing attribute data (BFS w/ LCA).
# - G Oldford, 2020

# gets cumulative passability each pair using path info
# and get other attributes

# pseudocode:
# for each 'from node' (e.g., sink in DCId, and all nodes in DCIp)
#  get paths between node and all other node
#
#  for each 'to node' in 'all paths' results
#   get the first edge len and hab traversed from node to sink
#
#   store length and hab of the edge between 'to node' and first node encountered
#   in path back to 'from node' (i.e., the 'to edge')

#   do while next node name <> "from node"
#     pass = nodeData(g_dd, nextnode, "pass")
#     cumulativepass =  cumulativepass * pass
#     nextnode = the next node in path towards 'from node'
#     if last edge traversed on the way to 'from node'
#       store the length and habitat of this edge which is the 'from edge'
#     if there is a maxdistance set for distance decay,
#       add a TRUE/FALSE column to indicate this
#
#
#   add various other attributes to master table (attr's from g object)

# requires library(data.table)
# data.table vs other options likely to speed things up for large networks
#https://rstudio-pubs-static.s3.amazonaws.com/406521_7fc7b6c1dc374e9b8860e15a699d8bb0.html
#https://www.rdocumentation.org/packages/data.table/versions/1.13.0/topics/rbindlist

get_summary_tab_2020 <- function(option="dt-lists",
                                 naturalonly=FALSE,
                                 g = NULL,
                                 DCIp=FALSE,
                                 bDistanceLim=FALSE,
                                 dMaxDist=0.0){

    # funciton params:
    # option - for benchmarking speed of appending to table
    # naturalonly - will calculate pass-weighted path distances
    #  while ignoring non-natural barriers
    # g - the graph object (rbgl GraphAM in BioconductR)
    # DCIp - TRUE / FALSE will trigger loop that finds path
    #       between all nodes, node just sink
    # initialize empty data object in different ways

    # for different options and benchmarking:
    DT2 = data.table(FromNode="init",
                 ToNode="init",
                 FromNodeLabel="init",
                 ToNodeLabel="init",
                 CumulativePass=0.0,
                 FromEdgeLen=0.0,
                 ToEdgeLen=0.0,
                 TotalDist=0.0,
                 DistMinusStartEndLen=0.0,
                 DistMinusSEExceedsThreshold=FALSE,
                 FromEdgeHab=0.0,
                 ToEdgeHab=0.0,
                 ToEdgeName="init",
                 FromEdgeName="init",
                 ToFromEdgeNameCombo="init")

    DF2 = data.frame(FromNode="init",
                 ToNode="init",
                 FromNodeLabel="init",
                 ToNodeLabel="init",
                 CumulativePass=0.0,
                 FromEdgeLen=0.0,
                 ToEdgeLen=0.0,
                 TotalDist=0.0,
                 DistMinusStartEndLen=0.0,
                 DistMinusSEExceedsThreshold=FALSE,
                 FromEdgeHab=0.0,
                 ToEdgeHab=0.0,
                 ToEdgeName="init",
                 FromEdgeName="init",
                 ToFromEdgeNameCombo="init", stringsAsFactors=F)

    # lists in R must have size pre-allocated
    # size of our table is almost n^2 - n*(n-1)
    # (less than n^2 since not getting distance from node to itself)
    if(DCIp==FALSE){
        outlist <- vector("list", length(numNodes(g_dd)))
    }else{
        outlist <- vector("list", length(numNodes(g_dd)*(numNodes(g_dd)-1)))
    }

    outlist[[numNodes(g_dd)]] <- list(FromNode="init",
                 ToNode="init",
                 FromNodeLabel="init",
                 ToNodeLabel="init",
                 CumulativePass=0.0,
                 FromEdgeLen=0.0,
                 ToEdgeLen=0.0,
                 TotalDist=0.0,
                 DistMinusStartEndLen=0.0,
                 DistMinusSEExceedsThreshold=FALSE,
                 FromEdgeHab=0.0,
                 ToEdgeHab=0.0,
                 FromEdgeName="init",
                 ToEdgeName="init",
                 ToFromEdgeNameCombo="init")

    # from = sink / start node
    # to = other nodes
    if(DCIp==FALSE){
        fromnodecount=1
    }else{
        fromnodecount=numNodes(g_dd)
    }

    bDistMinusSEExceedsThreshold = FALSE
    count = 0
    for (j in 1:fromnodecount){
    #indendation for two nested for loops not done to save space
        if(DCIp==FALSE){
            fromnode_name = "sink"
            fromnode_label = "sink"
        }else{
            fromnode_name = nodes(g_dd)[j]
             if (fromnode_name=="sink"){
                fromnode_label = "sink"
            }else{
                fromnode_label = nodeData(g_dd, fromnode_name, "nodelabel")[[1]]
            }
        }

        ###########################################################
        #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
        # get path & distances between 'fromnode' and all other nodes
        paths_distances <- get_paths_distances(g,fromnode_name)
        # this can be time consuming

    for (k in 1:length(paths_distances$penult)) {

        tonode <- paths_distances$penult[k]
        tonode_name = names(tonode)
        tonode_name <- tonode_name[[1]]
        tonode_label = nodeData(g_dd, tonode_name, "nodelabel")[[1]]

        if (tonode_name == fromnode_name){
            # not interested in distance from one node to itself
            next
        }
        count = count+1
        # initialize
        cumulativepass = 1.0
        pass = 1.0 # watch not to take pass from to/from end nodes since traversal starts at edge
        totaldistance = paths_distances$distances[tonode_name]
        totaldistance <- totaldistance[[1]]

        # get length of edge
        nextnode = paths_distances$penult[tonode]
        nextnode_name = names(nextnode)
        lastnode_name = tonode_name

        # get the last edge length traversed on the way to 'to node'
        # alternatively could grab the weight for this edge instead of subtraction
        toedgelen = totaldistance - paths_distances$distances[nextnode_name]
        toedgelen <- toedgelen[[1]]
        toedgedata =edgeData(g_dd, tonode_name,nextnode_name)
        toedgehab = toedgedata[[1]]$HabitatQuan
        toedgename = toedgedata[[1]]$EdgeNameGO

        exitvar = "go"
        while (exitvar != "stop"){

            if(nextnode_name != fromnode_name){
                pass = nodeData(g_dd, nextnode_name, "pass")
                if(naturalonly==FALSE){
                    cumulativepass = cumulativepass * pass[[1]]
                }else{
                    natural = nodeData(g_dd,nextnode_name,"natural")
                    if(natural[[1]]==TRUE){
                        cumulativepass = cumulativepass * pass[[1]]
                    }
                }
            }else{
                fromedgelen = paths_distances$distances[lastnode_name]
                fromedgelen <- fromedgelen[[1]]
                fromedgedata = edgeData(g_dd, lastnode_name,fromnode_name)
                fromedgehab = fromedgedata[[1]]$HabitatQuan
                fromedgename = fromedgedata[[1]]$EdgeNameGO

                exitvar="stop"
            }

            lastnode_name = nextnode_name
            nextnode = paths_distances$penult[nextnode]
            nextnode_name = names(nextnode)
        }

       distminusstartendlen = totaldistance - toedgelen - fromedgelen
       # less than zero distance indicates it's an edge-to-itself distance
       # correct for this
       if (distminusstartendlen<0){
          distminusstartendlen = 0
       }

       tofromedgename_combo = paste(toedgename,fromedgename,sep="|")

       if (bDistanceLim == TRUE){
           if (distminusstartendlen > dMaxDist){
               bDistMinusSEExceedsThreshold=TRUE
           }else{
               bDistMinusSEExceedsThreshold=FALSE
           }
       }else{
           bDistMinusSEExceedsThreshold=FALSE
       }

        if (option=="dt"){
            #print(cumulativepass)
            #https://www.rdocumentation.org/packages/data.table/versions/1.13.0/topics/rbindlist
            DT1 = data.table(FromNode=fromnode_name,
                     ToNode=tonode_name,
                     FromNodeLabel=fromnode_label,
                     ToNodeLabel=tonode_label,
                     CumulativePass=cumulativepass,
                     FromEdgeLen=fromedgelen,
                     ToEdgeLen=toedgelen,
                     TotalDist=totaldistance,
                     DistMinusStartEndLen=distminusstartendlen,
                     DistMinusSEExceedsThreshold = as.logical(bDistMinusSEExceedsThreshold),
                     FromEdgeHabLen=fromedgehablen,
                     ToEdgeHab=toedgehab,
                     FromEdgeHab=fromedgehab,
                     FromEdgeName=fromedgename,
                     ToEdgeName=toedgename,
                     ToFromEdgeNameCombo=tofromedgename_combo)
            l = list(DT1,DT2)

            DT2 = rbindlist(l, use.names=TRUE)
        }else if(option=="dt-lists"){
            # append lists to list rather than work yet with tables
            DL1 = list(FromNode=fromnode_name,
                     ToNode=tonode_name,
                     FromNodeLabel=fromnode_label,
                     ToNodeLabel=tonode_label,
                     CumulativePass=cumulativepass,
                     FromEdgeLen=fromedgelen,
                     ToEdgeLen=toedgelen,
                     TotalDist=totaldistance,
                     DistMinusStartEndLen=distminusstartendlen,
                     DistMinusSEExceedsThreshold = as.logical(bDistMinusSEExceedsThreshold),
                     FromEdgeHab=fromedgehab,
                     ToEdgeHab=toedgehab,
                     FromEdgeName=fromedgename,
                     ToEdgeName=toedgename,
                     ToFromEdgeNameCombo=tofromedgename_combo)
            #print("Length DL1: ")
            #print(length(DL1))
            outlist[[count]] <- (DL1)

        }else if(option=="df"){
            DF1 = data.frame(FromNode=fromnode_name,
                     ToNode=tonode_name,
                     FromNodeLabel=fromnode_label,
                     ToNodeLabel=tonode_label,
                     CumulativePass=cumulativepass,
                     FromEdgeLen=fromedgelen,
                     ToEdgeLen=toedgelen,
                     TotalDist=totaldistance,
                     DistMinusStartEndLen=distminusstartendlen,
                     DistMinusSEExceedsThreshold = as.logical(bDistMinusSEExceedsThreshold),
                     FromEdgeHab=fromedgehab,
                     ToEdgeHab=toedgehab,
                     FromEdgeName=fromedgename,
                     ToEdgeName=toedgename,
                     ToFromEdgeNameCombo=tofromedgename_combo)
            #print(fromedgehabarea)
            #print(DF1)
            DF2 <- rbind(DF2, DF1)
          }else if(option=="df-lists"){
            DL1 = list(FromNode=as.character(fromnode_name),
                     ToNode=as.character(tonode_name),
                     FromNodeLabel=as.character(fromnode_label),
                     ToNodeLabel=as.character(tonode_label),
                     CumulativePass=as.numeric(cumulativepass),
                     FromEdgeLen=as.numeric(fromedgelen),
                     ToEdgeLen=as.numeric(toedgelen),
                     TotalDist=as.numeric(totaldistance),
                     DistMinusStartEndLen=as.numeric(distminusstartendlen),
                     DistMinusSEExceedsThreshold = as.logical(bDistMinusSEExceedsThreshold),
                     FromEdgeHab=fromedgehab,
                     ToEdgeHab=toedgehab,
                     FromEdgeName=as.character(fromedgename),
                     ToEdgeName=as.character(toedgename),
                     ToFromEdgeNameCombo=as.character(tofromedgename_combo))
            outlist[[count]] <- (DL1)
        }else if(option=="dplyr"){
             DL1 = list(FromNode=as.character(fromnode_name),
                     ToNode=as.character(tonode_name),
                     FromNodeLabel=as.character(fromnode_label),
                     ToNodeLabel=as.character(tonode_label),
                     CumulativePass=as.numeric(cumulativepass),
                     FromEdgeLen=as.numeric(fromedgelen),
                     ToEdgeLen=as.numeric(toedgelen),
                     TotalDist=as.numeric(totaldistance),
                     DistMinusStartEndLen=as.numeric(distminusstartendlen),
                     DistMinusSEExceedsThreshold = as.logical(bDistMinusSEExceedsThreshold),
                     FromEdgeHab=as.numeric(fromedgehab),
                     ToEdgeHab=as.numeric(toedgehab),
                     FromEdgeName=fromedgename,
                     ToEdgeName=toedgename,
                     ToFromEdgeNameCombo=tofromedgename_combo)
            #print(row1)
            DF2 <- bind_rows(DF2,DL1)
        }
    } #k
    } #j

    if(option=="dt"){
        sum_tab <- DT2
    }else if(option=="dt-lists"){
        DT2 <- data.table(rbindlist(outlist))

        sum_tab <- DT2
    }else if(option=="df"){
        #DF2 <- DF2[!duplicated(DF2$ToFromEdgeNameCombo), ]
        sum_tab <- DF2
    }else if(option=="df-lists"){
        DF2 <- data.frame(do.call(rbind, outlist))
        #DF2 <- DF2[!duplicated(DF2$ToFromEdgeNameCombo), ]
        sum_tab <- DF2
    }else if(option=="dplyr"){
        sum_tab <- DF2
    }

    # if a distance limit, eliminate rows
    if(bDistanceLim == TRUE){
        return(sum_tab[DistMinusSEExceedsThreshold==FALSE])
    }else{
        return(sum_tab)
    }

}

#######################################################
###### Calculate DCI #####
#dci_calc_2020_dd <- function(){}
# warning: the sum_tab_2020 must be a data.table
#          only some data frames work ('dplyr' is ok not
#          the 'df-lists' option)

######### (1) DCId - calc_DCId #########
calc_DCId_2020 <- function(sum_tab_2020=NULL,
                           totalhabitat=0.0,
                           FromNode="sink",
                           bDistanceLim=FALSE){

    # filter table
    DCId_data<-subset(sum_tab_2020,FromNode=="sink")

    # Credit to Chris Edge Code for avoiding loops in R here:
    if(bDistanceLim==FALSE){
        DCId_data$temp <- DCId_data$CumulativePass * (DCId_data$ToEdgeHab/totalhabitat)
    }else{
        DCId_data$temp <- DCId_data$CumulativePass * (DCId_data$ToEdgeHabMaxAccessible/DCId_data$MaxTotalAccessHabFromEdge)
    }
    DCId <- sum(DCId_data$temp)
    return(round(DCId*100,2))
} # DCId

######### (2) DCIp - calc_DCIp #########
# Credit to C Edge for shorter code
calc_DCIp_2020 <- function(sum_tab_2020=NULL,
                           totalhabitat=0.0,
                           option="unique",
                           bDistanceLim=FALSE){

    # 'option' = unique / distinct for benchmarking speeds
    # unique = data.table / base r
    # distinct = dplyr
    # to do: sometimes sum_tab may be a data.table sometimes data.frame
    if(option=="unique"){
        sum_tab_2020 <- unique(sum_tab_2020, by = "ToFromEdgeNameCombo")
    }else{
        sum_tab_2020 <- distinct(sum_tab_2020, ToFromEdgeNameCombo, .keep_all = TRUE)
    }

    DCIp <- 0
    if(bDistanceLim==FALSE){
        for (i in 1:nrow(sum_tab_2020)) {
            DCIp <- DCIp + (sum_tab_2020$CumulativePass[i] * (sum_tab_2020$FromEdgeHab[i]/totalhabitat)) * (sum_tab_2020$ToEdgeHab[i]/totalhabitat)
        }
    }else{
        for (i in 1:nrow(sum_tab_2020)) {
            DCIp <- DCIp + (sum_tab_2020$CumulativePass[i] * (sum_tab_2020$FromEdgeHab[i]/totalhabitat))* (sum_tab_2020$ToEdgeHabMaxAccessible[i]/sum_tab_2020$MaxTotalAccessHabFromEdge[i])
        }
    }
    return(round(DCIp*100,2))
}

######### (3) DCIs - calc_DCIs #########
# can be added into (2) as option see below
calc_DCIs_2020 <- function (sum_tab_2020=NULL,
                            totalhabitat=0.0,
                            option="dt",
                            bDistanceLim=FALSE){

    # option: "dt","dplyr","old"
    # alternative methods for benchmarking
    if(option=="dt"){

        DCIs <- sum_tab_2020
        # remove duplicates
        DCIs <- unique(sum_tab_2020, by = "ToFromEdgeNameCombo")

        if(bDistanceLim==FALSE){
            # select only a required columns
            cols = c("FromEdgeName","ToEdgeHab","CumulativePass")
            DCIs <- DCIs[,..cols]
            # first step to DCIs
            DCIs[, DCIs_i := round(ToEdgeHab/totalhabitat * CumulativePass * 100,2)]
        }else{
            # select only a required columns
            cols = c("FromEdgeName","ToEdgeHabMaxAccessible","CumulativePass","MaxTotalAccessHabFromEdge")
            DCIs <- DCIs[,..cols]
            # first step to DCIs
            DCIs[, DCIs_i := round(ToEdgeHabMaxAccessible/MaxTotalAccessHabFromEdge * CumulativePass * 100,2)]
        }

        cols = c("FromEdgeName","DCIs_i")
        DCIs <- DCIs[,..cols]
        # second step to DCIs
        DCIs <- DCIs[, lapply(.SD,sum), by=.(FromEdgeName)]

    }else if(option=="dplyr"){

        if(bDistanceLim==FALSE){
            DCIs <- sum_tab_2020 %>%
            distinct(ToFromEdgeNameCombo, .keep_all = TRUE) %>%
            mutate(DCIs_i = CumulativePass * ToEdgeHab/totalhabitat * 100) %>%
            select(DCIs_i,FromEdgeName) %>%
            group_by(FromEdgeName) %>%
            summarise(DCIs = sum(DCIs_i))
        }else{
            DCIs <- sum_tab_2020 %>%
            distinct(ToFromEdgeNameCombo, .keep_all = TRUE) %>%
            mutate(DCIs_i = CumulativePass * ToEdgeHabMaxAccessible/MaxTotalAccessHabFromEdge)*100 %>%
            select(DCIs_i,FromEdgeName) %>%
            group_by(FromEdgeName) %>%
            summarise(DCIs = sum(DCIs_i))
        }

    }else if(option=="old"){
        sum_tab_2020 <- unique(sum_tab_2020, by = "ToFromEdgeNameCombo")
        sections<-as.vector(unique(sum_tab_2020$FromEdgeName))
        # store the all section results in DCI.as
        DCI_as<-NULL

        for(s in 1:length(sections)){
            DCI_s<-0
            # Old notes:
            # select out only the data that corresponds to pathways from one sectino
            # to all other sections
            d_nrows<-subset(sum_tab_2020, FromEdgeName==sections[s])
            d_sum_table<-d_nrows

            if(bDistanceLim==FALSE){
                for (a in 1:dim(d_nrows)[1]){
                    # Old note:
                    #to get the DCI for diadromous fish, use the following formula:
                    # DCId= li/L*Cj (where j= the product of the passability in the pathway)
                    la<-d_sum_table$ToEdgeHab[a]/sum(FIPEX_table$HabQuantity)
                    pass_d<-d_sum_table$CumulativePass[a]
                    DCI_s<-round(DCI_s+la*pass_d*100, digits=2)
                } # end loop over sections for dci calc
            }else{
                for (a in 1:dim(d_nrows)[1]){
                    # Old note:
                    #to get the DCI for diadromous fish, use the following formula:
                    # DCId= li/L*Cj (where j= the product of the passability in the pathway)
                    la<-d_sum_table$ToEdgeHabMaxAccessible[a]/d_sum_table$MaxTotalAccessHabFromEdge
                    pass_d<-d_sum_table$CumulativePass[a]
                    DCI_s<-round(DCI_s+la*pass_d*100, digits=2)
                } # end loop over sections for dci calc
            }
            DCI_as[s]<-round(DCI_s*100,2)
        } # end loop over "first" sections

        # STORE RESULTS IN .CSV file
        DCIs<-data.frame(sections,DCI_as)
    }else{
        print("error in options passed to calc_DCIs")
        DCIs <- 0.0
    }
    return(DCIs)
}

apply_distance_limits <- function(sum_tab_2020 = NULL,
                                bDistanceLim=FALSE,
                                dMaxDist=0.0,
                                bDistanceDecay=FALSE,
                                sDDFunction="none"){

    # calculate proportion of 'to edge' within the max distance and multiply
    #sum_tab_2020 <- sum_tab_2020 %>%
    #mutate(ToEdgeHabProp = (DistMinusStartEndLen+ToEdgeLen-dMaxDist)/ToEdgeLen) %>%
    #mutate(ToEdgeHabProp = ifelse(ToEdgeHabProp>=0,1-ToEdgeHabProp,1)) %>%
    #mutate(ToEdgeHabMaxAccessible = ToEdgeHabProp * ToEdgeHab) %>%
    #select(-ToEdgeHabProp)

    get_max_accessible <- function(DistMinusStartEndLen,ToEdgeLen,ToEdgeHab,dMaxDist){
        prop_accessible = (dMaxDist-DistMinusStartEndLen)/ToEdgeLen
        prop_accessible = ifelse(prop_accessible>0,prop_accessible,0)
        prop_accessible = ifelse(prop_accessible>1,1,prop_accessible)
        toedge_maxaccessible = prop_accessible * ToEdgeHab
        toedge_maxaccessible
    }

    sum_tab_2020[, ToEdgeHabMaxAccessible := get_max_accessible(DistMinusStartEndLen,
                                                                ToEdgeLen,ToEdgeHab,dMaxDist)]

    if(bDistanceDecay==TRUE & sDDFunction!="none"){
        sum_tab_2020 <- apply_distance_decay(sum_tab_2020,sDDFunction,dMaxDist)
        # overwriting here because totals below should be based on weighted hab
        sum_tab_2020$ToEdgeHabMaxAccessible = sum_tab_2020$toedgehabaccessible_dd
    }

    # sum habitat accessible from edge and add as attribute
    # in new column
    # remove duplicates first...
        # to do: dplyr is slower than data.table - re-code and benchmark
    sum_tab_hab <- sum_tab_2020 %>%
    distinct(ToFromEdgeNameCombo, .keep_all = TRUE) %>%
    group_by(FromEdgeName) %>%
    summarise(MaxTotalAccessHabFromEdge = sum(ToEdgeHabMaxAccessible))

    sum_tab_2020 <- sum_tab_2020 %>%
    left_join(sum_tab_hab, by = "FromEdgeName", copy=FALSE)

    return(data.table(sum_tab_2020))
}

apply_distance_decay <- function(sum_tab_2020=NULL,
                              sDDFunction="none",
                              dMaxDist = 0.0){

# distance decay options: "linear" - linear (1-x),
#                         "natexp1"- natural exponential #1 (general form: e^x),
#                         "natexp2" - natural exponential #2 (general form: e^x^2),
#                         "circle" - based on equation of circle ((1-x^2)^0.5)
#                         "sigmoid" - sigmoid (general form:1/(1+e^x)
# functions chosen because they can be integrated analytically.
# for analytical solutions see documentation.
# general form modified so intercepts are (0,1),(1,0) - sometimes approximate
# G Oldford, 2020

# to do: data.tables with many columns don't perform well and melt() may
#        help

########## General Formulas ##########
#  multiplies the 'maximum accessible habitat' at edge j by f_avg(a,b)
# (max accessible is pre-calculated earlier using cut-off value and distance of edge j from edge i)
#
# avg value of a dd function:
# f_avg(a,b,f(x)) = 1/(b-a)*integral_a_to_b(f(x)dx)
#           represents average value of distance decay function f(x) between
#           two positions, a and b, along total distance from end of edge i to maxdist
#           where a and b are positions from 0 to maxdist (maxdist is always
#           rescaled to 1.
# 'a' - proportion of maxdist reached at start of edge j
# 'b' - proportion of maxdist reached at end of edge j
    e = exp(1)

########## LINEAR distance decay function ##########
# function: (1-x) where x is proportion of maxdist
# f_avg = ((1-b)+(1-a))/2
# toedgehabaccessible_dd = f_avg(a,b)*ToEdgeHabMaxAccessible
    f_avg_linear <- function(a,b) {((1-b)+(1-a))/2}

########## natural exponential DD Function #1 ##########
########## general form: (-e^x) ##########
# parameterized function: f(x) = 1-e^(5(x-5)) with zero intercepts (0,1),(1,0)
# integral of f(x) = F_int = x - e^(5(x-1))/5)
# f_avg(a,b,f(x)) = 1/(b-a)*(b - e^(5(b-1))/5 -(a - e^(5(a-1))/5))
    integral_fx_natexp1 <- function(x){x-e^(5*(x-1))/5}
    f_avg_natexp1 <- function(a,b) {(1/(b-a)*(integral_fx_natexp1(b)-integral_fx_natexp1(a)))}

########## natural exponential DD Function #2 ##########
########## general form: -e^x^2 ##########
# parameterized function: f(x) = 2 - e^((x^2)*1/1.44)
# integral of f(x) = F_int = ((pi*i)^1/3 * erf(5*i*x/6)) / 5 + 2*x
# f_avg(a,b,f(x)) = to do
    # Note complete - can't be done without special erfi function
    erf <- function(x) 2 * pnorm(x * sqrt(2)) - 1
    integral_fx_natexp2 <- function(x){((pi*i)^1/3 * erf(5*i*x/6)) / 5 + 2*x}

########## CIRCULAR DD Function ##########
########## general form: (1-x^2)^0.5  ##########
# parameterized function: f(x) = (1-x^2)^0.5
# integral of f(x) = F_int = (asin(x)+x*(1-x^2)^0.5)/2
# f_avg(a,b,f(x)) = (asin(b)+b*(1-b^2)^0.5)/2 - (asin(a)+a*(1-a^2)^0.5)/2
    integral_fx_circle <- function(x){(asin(x)+x*(1-x^2)^0.5)/2}
    f_avg_circle <- function(a,b){((asin(b)+b*(1-b^2)^0.5)/2 - (asin(a)+a*(1-a^2)^0.5)/2)}

###### SIGMOID DD function ######
###### general form: 1/(e^x+1))######
# parameterized function: f(x) = 1/(e^(10(x*0.5)))+1
# integral f(x) = F_int = x - ln(e^10x+e^5)/10
# f_avg(a,b,f(x)) = 1/(b-a)*(b - ln(e^10b+e^5)/10- a - ln(e^10a+e^5)/10)
    integral_fx_sigmoid <- function(x){x - log(e^10*x+e^5)/10}
    f_avg_sigmoid <- function(a,b) {(1/(b-a)*(integral_fx_sigmoid(b)-integral_fx_sigmoid(a)))}

##### Calculate distance-decay weighted habitat accessible ######
##### (at each edge j from each each i)
    get_dd_habitat <- function(DistMinusStartEndLen,ToEdgeHabMaxAccessible,dMaxDist,sDDFunction){

        a = round(DistMinusStartEndLen/dMaxDist,3)
        b = round((DistMinusStartEndLen+ToEdgeHabMaxAccessible)/dMaxDist,3)

        a = ifelse(a<0,0,a)
        a = ifelse(a>1,1,a)
        b = ifelse(b<0,0,b)
        b = ifelse(b>1,1,b)

        if(sDDFunction=="linear"){
            toedgehab_dd <- ToEdgeHabMaxAccessible * f_avg_linear(a,b)
        }else if(sDDFunction=="natexp1"){
            toedgehab_dd <- ToEdgeHabMaxAccessible*f_avg_natexp1(a,b)
        }else if(sDDFunction=="natexp2"){
            # to do
        }else if(sDDFunction=="circle"){
            toedgehab_dd <- ToEdgeHabMaxAccessible*f_avg_circle(a,b)
        }else if(sDDFunction=="sigmoid"){
            toedgehab_dd <- ToEdgeHabMaxAccessible*f_avg_sigmoid(a,b)
        }

        # if the segment is tiny compared to cutoff distance
        # this can result in NaN's which cause issues.
        toedgehab_dd[ is.nan(toedgehab_dd) ] <- 0

        toedgehab_dd
    }

    sum_tab_2020[, toedgehabaccessible_dd := round(get_dd_habitat(DistMinusStartEndLen,
                                                                   ToEdgeHabMaxAccessible,
                                                                   dMaxDist,
                                                                   sDDFunction),2)]
    return(sum_tab_2020)

}

# ########################################
# ########## MAIN CODE SECTION ############

# intialize file with error code
write("ERROR",file='out_dd.txt')

# required libraries
library(RBGL)
library(data.table)
library(tidyverse)
# RBGL and Rgraphviz must be installed via BioconductR
#install.packages("BiocManager")
#BiocManager::install("Rgraphviz")
#BiocManager::install("RBGL")

# optional libraries
#library(Rgraphviz) # only needed for visuals
#library(rbenchmark)

########## 1) DATA PREP #########

# 'Advanced' table includes habitat, length, connectivity info in one table
FIPEX_table=read.csv("FIPEX_Advanced_DD_2020.csv")
# ensure it's "sink" not "Sink"
FIPEX_table <- FIPEX_table %>%
mutate(DownstreamEID = ifelse(DownstreamEID == "Sink","sink",as.character(DownstreamEID)))

# for testing only: natural TF set node 84 to natural barrier
#FIPEX_table[2,]$NaturalTF <- TRUE
#FIPEX_table

# The params file allows users to pass param settings
# to R from within the ArcMap software (new in 2020)
FIPEX_params=read.csv("FIPEX_2020_params.csv")

###### 2) PARAMETERIZATION #######

bDCISectional <- as.logical(FIPEX_params$bDCISectional)
bDistanceLim <- as.logical(FIPEX_params$bDistanceLim)
dMaxDist <- as.double(FIPEX_params$dMaxDist)
bDistanceDecay <- as.logical(FIPEX_params$bDistanceDecay)
sDDFunction <- as.character(FIPEX_params$sDDFunction)
# for testing:
#sDDFunction = "linear"
#sDDFunction = "natexp1"
#sDDFunction = "circle"
#sDDFunction = "sigmoid"
#sDDFunction = "none"
#bDistanceLim = FALSE
#dMaxDist = 1000
#bDistanceDecay = FALSE
naturalonly = FALSE
#bDCISectional = TRUE
bDCIp = TRUE

totalhabitat = sum(FIPEX_table$HabQuantity)
totallength = sum(FIPEX_table$DownstreamNeighDistance)

######### 3) NETWORK ANALYSIS #########

# build adjacency matrix
edgeweighted_adj_matrix <- create_advanced_adjmatrix_2020(FIPEX_table)

# build graph object
g_dd <- create_graph_dd_2020(edgeweighted_adj_matrix,FIPEX_table)

# build summary table (analyses to determine paths, pass, etc between edges)
sum_tab_2020 <- get_summary_tab_2020(option="dt-lists",
                                     naturalonly,
                                     g_dd,
                                     bDCIp,
                                     bDistanceLim,
                                     dMaxDist)

if(bDistanceLim==TRUE){
    sum_tab_2020 <- apply_distance_limits(sum_tab_2020, bDistanceLim, dMaxDist, bDistanceDecay, sDDFunction)
}

# for testing: turn all pass = 1 and DCI should = 1
#sum_tab_2020<- sum_tab_2020 %>% mutate(CumulativePass=1)
#sum_tab_2020 = as.data.table(sum_tab_2020)

######## 4) DCI CALC ########
DCId = 0.00
DCIp = 0.00
DCIs = 0.00

DCId <- calc_DCId_2020(sum_tab_2020,totalhabitat,"sink",bDistanceLim)
if(bDCIp==TRUE){
    DCIp <- calc_DCIp_2020(sum_tab_2020,totalhabitat,"unique",bDistanceLim)
	# sectional can only be run if DCIp has been selected
    if(bDCISectional==TRUE){
        DCIs <- calc_DCIs_2020(sum_tab_2020,totalhabitat,"dt",bDistanceLim)
    }
}

######## 5) Write to Files ######

# following previous output format
res<- data.frame(c(DCIp,DCId))
names(res)<-"value"
row.names(res)<-c("DCIp","DCId")
write.table(res,file='out_dd.txt')

# transform data to match what FIPEX expects
# DCIs 'FromEdgeName' 100-101 has first numbers as downstream node
# to align in FIPEX it can be adjusted to e.g., 100_s
# however, this results in more than one segment since, say, node
# 100 can have multiple upstream edges and nodes.
# To address this the _downstream_ segment DCI_s can be reported
# for each node in the system. This change will have to be carefully reported
# to the user!
if(bDCISectional==TRUE & bDCIp==TRUE){
    names(DCIs)[names(DCIs) == 'DCIs_i'] <- 'DCI_as'
    names(DCIs)[names(DCIs) == 'FromEdgeName'] <- 'section'

    #DCIs$sections <- paste(sub("\\-.*", "", DCIs$section),"_s",sep="")
    # there was a problem encountered with the above because it resulted
    # in more than one upstream segment associated with each node. I
    # reversed this but now need to be careful that the segmental DCI
    # is now reported as associated with the immediate _downstream_
    # segement from each node!
    DCIs$sections <- paste(sub(".*\\-", "", DCIs$section),"_s",sep="")

    DCIs$sections[DCIs$sections == "sink_s"] <- "sink"
    DCIs <- DCIs %>% select(sections,DCI_as)
    res<-data.frame(DCIs)
    write.table(x=res,
                file="DCI_all_sections_dd.csv",
                sep=",",
                row.names=F)
}