#' Drawing a sample from a fitted joint distribution
#'
#' @description
#' This function draws a random sample from an existing fitted Weibull-log-normal
#' (\code{wln}) or Heffernan-Tawn (\code{ht}) model for a given duration.
#'
#' @param jdistr a \code{wln} or \code{ht} joint distribution object, as the output from function \code{\link{fit_wln}}
#' and \code{\link{fit_ht}} respectively.
#'
#' @param sim_year the period of the simulate data in equivalent number of years.
#'
#' @param perturbed_ht_residuals whether or not to the simulation from the Heffernan-Tawn (\code{ht}) model should
#' add a perturbation when sampling from the empirical residual distribution (default value is \code{TRUE}).  This
#' argument is ignored when the input object is a Weibull-log-normal (\code{wln}) distribution.
#'
#' @return The function returns \code{data.table} object with two columns - wave
#' height \code{hs} and wave period and \code{tp}.
#'
#' @examples
#'
#' # Load data
#' data(ww3_pk)
#'
#' # Fit Heffernan-Tawn model with 0.95 marginal and dependence thresholds
#' ht = fit_ht(data = ww3_pk, npy = nrow(ww3_pk)/10, margin_thresh_count = 100, dep_thresh_count = 100)
#'
#' # Draw the equivalent of 100-year of data from the fitted Heffernan-Tawn model with or without perturbed residuals
#' sim_ht1 = sample_jdistr(jdistr = ht, sim_year = 100, perturbed_ht_residuals = FALSE)
#' sim_ht2 = sample_jdistr(jdistr = ht, sim_year = 100)
#' par(mfrow=c(1,2))
#' plot(sim_ht1, pch=20, cex=.5, main="HT without perturbation")
#' plot(sim_ht2, pch=20, cex=.5, main="HT with perturbation")
#'
#' # Fit Weibull-log-normal model
#' wln = fit_wln(data = ww3_pk, npy = nrow(ww3_pk)/10)
#'
#' # Draw the equivalent of 100-year of data from the fitted Weibull-log-normal model
#' sim_wln = sample_jdistr(jdistr = wln, sim_year = 100)
#' plot(sim_wln, pch=20, cex=.5, main="WLN")
#'
#' @seealso \code{\link{fit_ht}}, \code{\link{fit_wln}}
#'
#' @export
sample_jdistr = function(jdistr, sim_year, perturbed_ht_residuals = TRUE){

  if(class(jdistr)=="ht"){
    res = .sample_ht(jdistr, sim_year, perturbed_ht_residuals)
  }else if(class(jdistr)=="wln"){
    res = .sample_wln(jdistr, sim_year)
  }else{
    stop("Input class of distribution not supported.")
  }
  return(res)
}

# Heffernan-Tawn ----------------------------------------------------------
.sample_ht = function(ht, sim_year, perturbed_ht_residuals){

  set.seed(.seed_sampling)
  n_sim = sim_year*ht$npy

  # Sample overall freq
  res = ht$dep$dep_data[sample.int(.N, size = n_sim, replace = T), .(u_hs, u_tp)]
  rot_wb = res[, .N^(-1/6)]
  res[, u_hs:=pnorm(qnorm(u_hs)+rnorm(.N, 0, rot_wb))]
  res[, u_tp:=pnorm(qnorm(u_tp)+rnorm(.N, 0, rot_wb))]
  n_hs = res[u_hs>ht$dep$p_dep_thresh & u_hs>u_tp, .N]
  n_tp = res[u_tp>ht$dep$p_dep_thresh & u_tp>=u_hs, .N]

  # Sample hs & tp extreme in unif
  dep_data_hs = .sample_ht1(n_hs, ht$dep$hs$par, ht$dep$hs$resid, ht$dep$p_dep_thresh, perturbed_ht_residuals)
  res[u_hs>ht$dep$p_dep_thresh & u_hs>u_tp, c("u_hs", "u_tp"):=dep_data_hs[, .(u_cond, u_dep)]]
  dep_data_tp = .sample_ht1(n_tp, ht$dep$tp$par, ht$dep$tp$resid, ht$dep$p_dep_thresh, perturbed_ht_residuals)
  res[u_tp>ht$dep$p_dep_thresh & u_tp>=u_hs, c("u_hs", "u_tp"):=dep_data_tp[, .(u_dep, u_cond)]]

  # Convert to original scale
  n = length(ht$margin$hs$emp)
  res[, hs:=.convert_unif_to_origin(
    unif = u_hs, p_thresh = ht$margin$p_margin_thresh, gpd_par = ht$margin$hs$par, emp = ht$margin$hs$emp)]
  res[, tp:=.convert_unif_to_origin(
    unif = u_tp, p_thresh = ht$margin$p_margin_thresh, gpd_par = ht$margin$tp$par, emp = ht$margin$tp$emp)]

  return(res[, .(hs, tp)])
}

.sample_ht1 = function(n_sim, par, resid, p_dep_thresh, perturbed_ht_residuals){
  set.seed(.seed_sampling)
  a = par[1]
  b = par[2]
  sim_data = data.table(u_cond=runif(n_sim, p_dep_thresh, 1))
  sim_data[, l_cond:=.convert_unif_to_lap(u_cond)]
  resid_max = sim_data[, (1-a)*l_cond^(1-b)]
  if(perturbed_ht_residuals){
    resid_bw = bw.SJ(resid)
    extended_resid = resid+rnorm(length(resid)*100, 0, resid_bw)
  }else{
    extended_resid = resid
  }
  resid_sample = sapply(resid_max, function(x)sample(extended_resid[extended_resid<x],1))
  sim_data[, l_dep:=resid_sample*l_cond^b+a*l_cond]
  sim_data[, u_dep:=.convert_lap_to_unif(l_dep)]
  return(sim_data[, .(u_cond, u_dep)])
}

.sample_ht_is = function(ht, target_rp){

  # Common
  n_sim = round(ht$npy*target_rp*.target_rp_ub)
  tail_prob = 1/(target_rp*.target_rp_lb*ht$npy)
  min_r = qnorm(1-tail_prob)
  n_tail = n_sim*exp(-min_r^2/2)

  # Generate importance samples outside the Rosenblatt transformed contour of tail_rtrp
  set.seed(.seed_sampling)
  calc = data.table(ur2 = runif(n_tail, min=pchisq(q = min_r^2,df = 2)))
  calc[, r:=sqrt(qchisq(p=ur2, df = 2))]
  calc[, dir:=runif(.N, min=0, max=2*pi)]

  # Generate by Tp|Hs model
  calc_hs = calc[, .(u_hs=pnorm(cos(dir)*r), u_tp_hs=pnorm(sin(dir)*r))]
  calc_hs[, lap_hs:=.convert_unif_to_lap(u_hs)]
  calc_hs[u_hs>=ht$dep$p_dep_thresh,lap_tp:=
            lap_hs*ht$dep$hs$par[["a"]]+lap_hs^ht$dep$hs$par[["b"]]*quantile(ht$dep$hs$resid, u_tp_hs)]
  calc_hs[u_hs>=ht$dep$p_dep_thresh, u_tp:=.convert_lap_to_unif(lap_tp)]
  calc_hs = calc_hs[u_hs>ht$dep$p_dep_thresh][u_hs>u_tp]
  calc_hs[, dir:=atan2(qnorm(u_hs), qnorm(u_tp))]

  # Generate by Tp|Hs model
  calc_tp = calc[, .(u_hs_tp=pnorm(cos(dir)*r), u_tp=pnorm(sin(dir)*r))]
  calc_tp[, lap_tp:=.convert_unif_to_lap(u_tp)]
  calc_tp[u_tp>=ht$dep$p_dep_thresh,lap_hs:=
            lap_tp*ht$dep$tp$par[["a"]]+lap_tp^ht$dep$tp$par[["b"]]*quantile(ht$dep$tp$resid, u_hs_tp)]
  calc_tp[u_tp>=ht$dep$p_dep_thresh, u_hs:=.convert_lap_to_unif(lap_hs)]
  calc_tp = calc_tp[u_tp>ht$dep$p_dep_thresh][u_tp>u_hs]
  calc_tp[, dir:=atan2(qnorm(u_hs), qnorm(u_tp))]

  # Generate low/low corner
  n_ll = calc[, .N]-calc_hs[, .N]-calc_tp[, .N]
  set.seed(.seed_sampling)
  res = ht$dep$dep_data[sample.int(.N, size = calc[, .N]/.target_rp_lb, replace = T), .(u_hs, u_tp)]
  # print(res[, .N])
  rot_wb = res[, .N^(-1/6)]
  res[, u_hs:=pnorm(qnorm(u_hs)+rnorm(.N, 0, rot_wb))]
  res[, u_tp:=pnorm(qnorm(u_tp)+rnorm(.N, 0, rot_wb))]
  res[, r:=sqrt(qnorm(u_hs)^2+qnorm(u_tp)^2)]
  res[, dir:=atan2(qnorm(u_hs), qnorm(u_tp))]
  res_ll = res[dir>max(calc_hs$dir) | dir<min(calc_tp$dir)]
  res_ll[, dir_grp:=round(dir, 3)]
  calc_ll = res_ll[, .SD[r>quantile(r, 1-n_ll/res_ll[,.N])], .(dir_grp)]

  # Merge
  out = rbind(calc_hs[, .(u_hs, u_tp)], calc_tp[, .(u_hs, u_tp)], calc_ll[, .(u_hs, u_tp)])
  out[, hs:=.convert_unif_to_origin(
    unif = u_hs, p_thresh = ht$margin$p_margin_thresh,
    gpd_par = ht$margin$hs$par, emp = ht$margin$hs$emp)]
  out[, tp:=.convert_unif_to_origin(
    unif = u_tp, p_thresh = ht$margin$p_margin_thresh,
    gpd_par = ht$margin$tp$par, emp = ht$margin$tp$emp)]

  return(out[, .(hs, tp)])
}

# Weibull log-normal ------------------------------------------------------
.sample_wln = function(wln, sim_year){
  set.seed(.seed_sampling)
  n_sim = round(wln$npy*sim_year)

  # Sample hs
  hs = rweibull(n_sim, shape = wln$hs$par["shape"], scale = wln$hs$par["scale"]) + wln$hs$par["loc"]

  # sample tp cond on hs
  tp_norm_mean = wln$tp$par[1] + wln$tp$par[2] * (hs ^ wln$tp$par[3])
  # tp_norm_sd = wln$tp$par[4] + wln$tp$par[5] * exp(wln$tp$par[6] * hs)
  tp_norm_sd = pmax(.limit_zero, wln$tp$par[4] + wln$tp$par[5] * exp(wln$tp$par[6] * hs))
  tp = exp(rnorm(n_sim, tp_norm_mean, tp_norm_sd))

  res = data.table(hs, tp)
  return(res)
}

.sample_wln_is = function(wln, target_rp){

  n_sim = round(wln$npy*target_rp*.target_rp_ub)
  tail_prob = 1/(target_rp*.target_rp_lb*wln$npy)
  min_r = qnorm(1-tail_prob)
  n_tail = n_sim*exp(-min_r^2/2)

  # Generate importance samples outside the Rosenblatt transformed contour of tail_rtrp
  set.seed(.seed_sampling)
  calc = data.table(ur2 = runif(n_tail, min=pchisq(q = min_r^2,df = 2)))
  calc[, r:=sqrt(qchisq(p=ur2, df = 2))]
  calc[, dir:=runif(.N, min=0, max=2*pi)]
  calc[, u_hs:=pnorm(cos(dir)*r)]
  calc[, u_tp:=pnorm(sin(dir)*r)]

  # Sample hs
  calc[, hs:= qweibull(u_hs, shape = wln$hs$par["shape"], scale = wln$hs$par["scale"]) + wln$hs$par["loc"]]

  # sample tp cond on hs
  calc[, tp_norm_mean:= wln$tp$par[1] + wln$tp$par[2] * (hs ^ wln$tp$par[3])]
  # calc[, tp_norm_sd:= wln$tp$par[4] + wln$tp$par[5] * exp(wln$tp$par[6] * hs)]
  calc[, tp_norm_sd:= pmax(.limit_zero, wln$tp$par[4] + wln$tp$par[5] * exp(wln$tp$par[6] * hs))]
  calc[, tp:= exp(qnorm(u_tp, tp_norm_mean, tp_norm_sd))]

  # Return
  return(calc[, .(hs, tp)])
}


#' @title
#' Calculate the ordering of events
#'
#' @description
#' For each event, we generate distributions of time of event occurance using profiles within clusters. These distributions are then pair wise compared and the results of all comparisons are represented in the graph space, where events are the graph nodes, and the directional edges represent that the source node event occurs significantly early than the target node. This graph is then utilized to generate the ordering.
#'
#' @param Tc A matrix containing time course data.
#' @param clusters A vector with same length as number of profiles. The cluster labels are assumed to be numbers, starting from 1.
#' @param mat_events A matrix containing the time regions (and events for the centroid) generated by running the \code{calcEvents} function.
#' @param test The test used to compare the event time distributions. The choice is between a parametric or a non-parametric test:
#' \describe{
#' 		\item{\code{wilcox}}{Choosing this will result in event times being compared using the Mann Whitney U test}
#' 		\item{\code{t-test}}{Choosing this will result in event times being compared using the Students t-test}
#' }
#' @param fdrSignif The significance cutoff for FDR correction, performed after pairwise comparisons of the event times.
#' @param phosEventTh Value between 0 and 1, to define the threshold at which a phosphorylation (or increasing) event occurs. Here, a threshold of 0 corresponds to the minimum value within an interval. (Such an event is designated as 1).
#' @param dephosEventTh Value between 0 and 1, to define the threshold at which a dephosphorylation (or decreasing) event occurs. A threshold of 0 corresponds to the maximum value within an interval. (Such an event is designated as -1).
#'
#'
#' @return A list with four named objects is returned, which encodes the order of the events and the clusters.
#'
#'
#' @importFrom methods is
#' @importFrom stats p.adjust wilcox.test t.test
#' @importFrom nem transitive.reduction
#' @importFrom igraph graph_from_adjacency_matrix adjacent_vertices V are.connected vertex_connectivity
#'
#' @seealso \code{\link[e1071]{cmeans}} for clustering time profiles, \code{\link{calcEvents}} for identifying events.
#'
#' @export
calculateOrder <- function(Tc, clusters, mat_events, test="wilcox", fdrSignif=0.05, phosEventTh=0.5, dephosEventTh=0.5){

	stopifnot(is(Tc, "matrix"),  (length(clusters) == nrow(Tc)), is(mat_events, "matrix"), (fdrSignif >0 && fdrSignif <= 1 ), (phosEventTh >= 0 && phosEventTh <= 1), (dephosEventTh >= 0 && dephosEventTh <= 1))

	if (!(test == eventOrderTest$param) && !(test == eventOrderTest$nonParam)){
		stop(paste("Test ", test, " not recognized.", sep=""))
	}

	list_matrices <- splitIntoSubMatrices(Tc, clusters)

	list_distributions <- getDistOfAllEvents_v2(mat_events, list_matrices, phosEventTh, dephosEventTh)

	# mat_missingStats <- missingStats(list_distributions) # exclude events based on percentage missing. mat_fiftyPoints, list_distributions
	# return(mat_missingStats)

	# return (list_distributions)

	list_pVal_stat <- list()

	if (test==eventOrderTest$nonParam){
		print ("Running wilcoxon test.")
		list_pVal_stat <- performWilcoxonSignedRankTabular(list_distributions)

		title_testType = "(Non-parametric)"
	}
	else {
		print("Running t-test.")
		list_pVal_stat <- performTtestsTabular(list_distributions)

		title_testType = "(Parametric)"
	}

	# FDR adjustment for multiple testing.
	mat_pValsFDR <- matrix(stats::p.adjust(as.vector(list_pVal_stat[[1]]), method='fdr'), ncol=nrow(mat_events))

	# Convert to adjacency matrix.
	signifs <- mat_pValsFDR < fdrSignif
	statistic_t <- list_pVal_stat[[2]]
	statistic_t[!signifs] <- NA

	statistic_t[statistic_t > 0] <- NA
	statistic_t[is.na(statistic_t)] <- 0
	statistic_t[statistic_t < 0] <- 1

	# library(nem) # for trasitive reduction
	# library(igraph) # for adjacency matrix
	reducted <- nem::transitive.reduction(statistic_t)
	net = igraph::graph_from_adjacency_matrix(reducted, mode="directed")

	list_eventsOrder <- getEventsOrderInGraph(reducted, net) # computing the order of the events in the mat_events based on a bfs search.
	list_eventsOrder <- adjustEvents(list_eventsOrder, signifs)

	mat_events_withOrder <- appendOrder(mat_events, list_eventsOrder)

	res <- list(mat_events_withOrder, list_eventsOrder, signifs, test)
	names(res) <- c("mat_events_withOrder", "individEventOrder", "signifs", "test")
	return (res)

}

#' @title
#' Visualize the order of events.
#'
#' @description
#' A plot depicting the temporal order of events is generated. The order is shown using two visualizations, event maps (top) and event sparklines (bottom). The event sparkline is a summary of the event map.
#'
#' @param theOrder A list containing information regarding the order of events and clusters generated via the \code{calculateOrder} function.
#'
#'
#' @return A plot containing the ordered clusters by occurance of first event is generated (see description).
#'
#'
#' @importFrom methods is
#' @importFrom graphics plot segments axis rect text lines
#' @importFrom shape Arrows
#'
#' @seealso \code{\link{calculateOrder}}
#'
#' @export
visualizeOrder <- function(theOrder){
# Visualise the ordering of events

	stopifnot(is(theOrder, "list"), length(theOrder) == 4)

	mat_events_withOrder <- theOrder$mat_events_withOrder
	list_eventsOrder <- theOrder$individEventOrder
	signifs <- theOrder$signifs
	test <- theOrder$test

	title_testType = ""
	## Getting other things ready for plotting

	if (test==eventOrderTest$nonParam){
		title_testType = "(Non-parametric)"
	}
	else {
		title_testType = "(Parametric)"
	}

	# plotting constants
	sigEventsDiff_x = 0.5
	nonSigEventsDiff_x = 0.25

	eventPtGraph_y = -1
	yLabelInit = -1.8
	yLabelSpace = 0.5

	lineDiff_y = 1

	# graphXStart = -1
	eventStart_x = 0

	blockSpacingFmEvent_x = sigEventsDiff_x/2
	blockStart_x = eventStart_x - blockSpacingFmEvent_x

	#x_lineStart, y_lineStart, y_lineDecr, x_incr_sig, x_incr_nonSig

	yPos_eventPts = -1

	# plotting values
	list_blocks <- getRectBlock(list_eventsOrder, signifs)
	# print(list_blocks)

	mat_eventPoints <- getTheClusLines(mat_events_withOrder, list_eventsOrder, signifs, list_blocks, eventStart_x, sigEventsDiff_x, nonSigEventsDiff_x, lineDiff_y)

	eventEnd_x = max(as.numeric(mat_eventPoints[,cols_matFifty$col_x])) #  + blockSpacingFmEvent_x
	blockEnd_x = eventEnd_x + blockSpacingFmEvent_x

	mat_bgRects <- getRectPoints(mat_eventPoints, list_blocks, blockStart_x, blockSpacingFmEvent_x)

	# mat_eventPoints <- adjustByBgRects(mat_bgRects, list_eventsOrder, mat_eventPoints)

	mat_grayLines <- getGrayLines(mat_eventPoints, signifs)

	mat_grayLines_v2 <- getGrayLines_v2(list_eventsOrder, mat_eventPoints, signifs)

	mat_clusConnLines <- getClusConnLines(mat_eventPoints, blockStart_x, blockEnd_x)

	mat_clusConnLines <- makeDecrBarOpaque(mat_clusConnLines, FALSE)
	# print(mat_clusConnLines)
	vec_labels <- getYaxisLabels(mat_eventPoints)

	mat_xLabelsClus <- getXaxisLabels(list_eventsOrder, mat_eventPoints, signifs, list_blocks, eventStart_x, sigEventsDiff_x, nonSigEventsDiff_x, yLabelInit, yLabelSpace);

	# a set of points to set up the graph.
	# print(mat_eventPoints)
	graphics::plot(mat_eventPoints[,cols_matFifty$col_x], mat_eventPoints[,cols_matFifty$col_y], asp=NA, yaxt="n", lwd=0.25, col=colors_orderedEvents$incr, pch=".", xlab="Temporal order", ylab="Clusters",  main=paste("Temporal order of events in clusters", title_testType, sep=""), xlim=c(eventStart_x, eventEnd_x), ylim=c(-4, max(as.numeric(mat_eventPoints[,cols_matFifty$col_clus]))), xaxt="n", bty="n", cex.main = 0.8) #,
	# print("over here..")
	# background gray rectangles.
	if (nrow(mat_bgRects) > 0){
		for (rowNum in 1:nrow(mat_bgRects)){
			if (rowNum%%2 == 1){
				graphics::rect(mat_bgRects[rowNum,1], lineDiff_y/2, mat_bgRects[rowNum,2], max(as.numeric(mat_eventPoints[,cols_matFifty$col_clus])) + (lineDiff_y/2), col = "#ededed", border=NA)
			}

		}
	}

	# axis labels on sides 2 and 4.
	graphics::axis(side=2, las=1, at=seq(1,length(vec_labels)), labels=rev(vec_labels), cex=0.02,  col = NA ) #, col.ticks = 1)

	graphics::axis(side=4, las=1, at=seq(1,length(vec_labels)), labels=rev(vec_labels), cex=0.02,  col = NA ) #, col.ticks = 1)

	graphics::segments(x0=as.numeric(mat_grayLines[,cols_clusPlotObjs$col_x0]), y0=as.numeric(mat_grayLines[,cols_clusPlotObjs$col_y0]), x1=as.numeric(mat_grayLines[,cols_clusPlotObjs$col_x1]), y1=as.numeric(mat_grayLines[,cols_clusPlotObjs$col_y1]), col="#808080", lty="dotted", lwd=0.5)
	# C99999

	# rect(-0.5, 0, 0.5, 17.5, col = "yellow", border="#ededed")

	# the cluster lines
	graphics::segments(x0=as.numeric(mat_clusConnLines[,cols_clusPlotObjs$col_x0]), y0=as.numeric(mat_clusConnLines[,cols_clusPlotObjs$col_y0]), x1=as.numeric(mat_clusConnLines[,cols_clusPlotObjs$col_x1]), y1=as.numeric(mat_clusConnLines[,cols_clusPlotObjs$col_y1]), col=mat_clusConnLines[,cols_clusPlotObjs$col_col], lwd=16, lend=2)

	# the events (arrows) within a graph
	mat_upEvents = mat_eventPoints[mat_events_withOrder[,cols_matFifty$col_dir] == 1,]
	mat_downEvents = mat_eventPoints[mat_events_withOrder[,cols_matFifty$col_dir] == -1,]

	# print(mat_upEvents)

	## arrow bottoms
	shape::Arrows(x0=as.numeric(mat_upEvents[,cols_matFifty$col_x]), x1=as.numeric(mat_upEvents[,cols_matFifty$col_x]), y0=as.numeric(mat_upEvents[,cols_matFifty$col_y])-0.01-0.2,  y1=as.numeric(mat_upEvents[,cols_matFifty$col_y])-0.01, arr.type="triangle", arr.length=0.01, arr.width=0.01, col=color_events$up, lwd=2, lend=2, arr.adj=0)

	shape::Arrows(x0=as.numeric(mat_downEvents[,cols_matFifty$col_x]), x1=as.numeric(mat_downEvents[,cols_matFifty$col_x]), y0=as.numeric(mat_downEvents[,cols_matFifty$col_y])+0.25,  y1=as.numeric(mat_downEvents[,cols_matFifty$col_y])+0.25-0.2, arr.type="triangle", arr.length=0.01, arr.width=0.01, col=color_events$down, lwd=2, lend=2, arr.adj=0)

	## sharp arrow heads
	shape::Arrows(x0=as.numeric(mat_upEvents[,cols_matFifty$col_x]), x1=as.numeric(mat_upEvents[,cols_matFifty$col_x]), y0=as.numeric(mat_upEvents[,cols_matFifty$col_y])-0.01-0.2,  y1=as.numeric(mat_upEvents[,cols_matFifty$col_y])-0.01, arr.type="triangle", arr.length=0.13, arr.width=0.15, col=color_events$up, arr.adj=0, segment=FALSE)

	shape::Arrows(x0=as.numeric(mat_downEvents[,cols_matFifty$col_x]), x1=as.numeric(mat_downEvents[,cols_matFifty$col_x]), y0=as.numeric(mat_downEvents[,cols_matFifty$col_y])+0.25,  y1=as.numeric(mat_downEvents[,cols_matFifty$col_y])+0.25-0.2, arr.type="triangle", arr.length=0.13, arr.width=0.15, col=color_events$down, arr.adj=0, segment=FALSE)

	# gray lines in events points maps
	straightGrayLines = mat_grayLines_v2[mat_grayLines_v2[,cols_grayLines_v2$col_isSemiCirc] == FALSE,]
	graphics::segments(x0=as.numeric(straightGrayLines[,cols_grayLines_v2$col_x0]), y0=rep(yPos_eventPts, length(straightGrayLines)), x1=as.numeric(straightGrayLines[,cols_grayLines_v2$col_x1]), y1=rep(yPos_eventPts, length(straightGrayLines)), lwd=1, col="#A2A2A2") #, lty="dotted")

	# gray curves in events points map
	mat_curvePoints = mat_grayLines_v2[mat_grayLines_v2[,cols_grayLines_v2$col_isSemiCirc] == TRUE,,drop=FALSE]

	if (length(mat_curvePoints) > 0 && nrow(mat_curvePoints) > 0){
		for (i in 1:nrow(mat_curvePoints)){
			list_pts = calcCurves(mat_curvePoints[i, cols_grayLines_v2$col_x0], mat_curvePoints[i, cols_grayLines_v2$col_x1], -1)
			graphics::lines(list_pts[[1]], list_pts[[2]], col="#A2A2A2")
		}
	}

	# printing up and down events (in arc events map)
	y_adjust = 0.15

	## arrow bottom lines
	shape::Arrows(x0=as.numeric(mat_upEvents[,cols_matFifty$col_x]), x1=as.numeric(mat_upEvents[,cols_matFifty$col_x]), y0=rep(-1.2, nrow(mat_upEvents)), y1=rep(-1, nrow(mat_upEvents)), arr.type="triangle", arr.length=0.01, arr.width=0.01, col=color_events$up, lwd=2, lend=2, arr.adj=0)

	shape::Arrows(x0=as.numeric(mat_downEvents[,cols_matFifty$col_x]), x1=as.numeric(mat_downEvents[,cols_matFifty$col_x]), y0=rep(-0.8, nrow(mat_downEvents)), y1=rep(-1, nrow(mat_downEvents)), arr.type="triangle", arr.length=0.01, arr.width=0.01, col=color_events$down, lwd=2, lend=2, arr.adj=0)

	## sharp arrow heads
	shape::Arrows(x0=as.numeric(mat_upEvents[,cols_matFifty$col_x]), x1=as.numeric(mat_upEvents[,cols_matFifty$col_x]), y0=rep(-1.2, nrow(mat_upEvents)), y1=rep(-1, nrow(mat_upEvents)), arr.type="triangle", arr.length=0.13, arr.width=0.15, col=color_events$up, arr.adj=0, segment=FALSE)

	shape::Arrows(x0=as.numeric(mat_downEvents[,cols_matFifty$col_x]), x1=as.numeric(mat_downEvents[,cols_matFifty$col_x]), y0=rep(-0.8, nrow(mat_downEvents)), y1=rep(-1, nrow(mat_downEvents)), arr.type="triangle", arr.length=0.13, arr.width=0.15, col=color_events$down, arr.adj=0, segment=FALSE)

	# vec_clusEvent = mat_eventPoints[,4]
	# print(vec_clusEvent)
	# vec_clusEvent[mat_eventPoints[mat_fiftyPoints[,cols_matFifty$col_dir] == 1]] = color_events$up
	# vec_clusEvent[mat_eventPoints[mat_fiftyPoints[,cols_matFifty$col_dir] == -1] = color_events$down

	# x label text.
	graphics::text(x=as.numeric(mat_xLabelsClus[,1]), y=as.numeric(mat_xLabelsClus[,2]), labels=mat_xLabelsClus[,3], offset=0, cex=0.5, col=mat_xLabelsClus[,4])

}

#' @title
#' Rearrange clusters by events
#'
#' @description
#' After calculating the order of events, we can now rearrange clusters, so that the first cluster contains the first event and so on.
#'
#' @param clusters A vector with same length as number of profiles. The cluster labels are assumed to be numbers, starting from 1.
#' @param theOrder A list containing information regarding the order of events and clusters generated via the \code{calculateOrder} function.
#'
#'
#' @return The returned vector now contains the cluster rearranged according to the computed ordering in \code{theOrder}.
#'
#'
#' @importFrom methods is
#' @importFrom rlang duplicate
#'
#' @seealso \code{\link[e1071]{cmeans}} for clustering, \code{\link{calculateOrder}} to generate the order.
#'
#' @export
rearrangeClusters <- function(clusters, theOrder){

	stopifnot(is(theOrder, "list"), length(theOrder) == 4)

	clusterOrder <- matrix(ncol=1, nrow=max(theOrder$mat_events_withOrder[,cols_matFifty_v2$clus]), data=NA) # the row numb stores the original order
	# print(clusterOrder)
	orderCounter = 1
	for (i in 1:length(theOrder$individEventOrder)){
		for (j in 1:length(theOrder$individEventOrder[[i]])){

			clusNum = theOrder$mat_events_withOrder[theOrder$individEventOrder[[i]][j], cols_matFifty_v2$clus]

			if (is.na(clusterOrder[clusNum, 1])){
				# print(paste(clusNum, orderCounter))
				clusterOrder[clusNum, 1] <- orderCounter;

				orderCounter = orderCounter + 1;
				# print(clusterOrder)
			}
		}
	}
	# return(clusterOrder)

	origClusters = clusters
	# centers = rlang::duplicate(clustered$centers)
	for (origOrder in 1:nrow(clusterOrder)){
		if (!is.na(clusterOrder[origOrder,1])){
			# print(paste(origOrder, clusterOrder[origOrder,1]))
			clusters[origClusters == origOrder] = clusterOrder[origOrder,1]
			# clustered$center[clusterOrder[origOrder,1], ] = centers[origOrder,]
		}
	}
	return (clusters)
}

calcCurves <- function(x0, x1, yCentPos){

	x_10pts <- seq(x0, x1, length.out=50)

	y_10pts <- topSemiCircleEq(x_10pts, yCentPos)

	list_pts <- list(x_10pts, y_10pts)
	return (list_pts)

}

topSemiCircleEq <- function(xPts, yCentPos){
	yPts = c()

	r = abs(xPts[1] - xPts[length(xPts)])/2
	xCentPos = (xPts[1] + xPts[length(xPts)]) /2

	for (xPt in xPts){
		yPt = sqrt(abs(r^2 - (xPt-xCentPos)^2)) + yCentPos + 0.4
		yPts = c(yPts, yPt)
	}

	return (yPts)
}

appendOrder <- function(mat_fiftyPoints, list_eventsOrder){

	# mat_withOrder <- cbind(mat_fiftyPoints, rep(0,nrow(mat_fiftyPoints)))

	for (i in 1:length(list_eventsOrder)){
		eventNum <- list_eventsOrder[[i]]

		mat_fiftyPoints[eventNum, Col_events$order] = i
	}

	colnames(mat_fiftyPoints)[Col_events$order] = "order";
	return(mat_fiftyPoints)
}

################################## AUXILLARY
adjustEvents <- function(list_eventsOrder, signifs){
	list_newEventsOrder <- list()
	i_new = 1

	for (i in 1:length(list_eventsOrder)){
		# print(list_eventsOrder[[i]])
		isAddedVec = FALSE
		for (j in 1:length(list_eventsOrder[[i]])){

			if (isAddedVec == FALSE){ # add every row atleast once
				# add to list first
				list_newEventsOrder[[length(list_newEventsOrder) + 1]] <- list_eventsOrder[[i]]
				isAddedVec = TRUE
				# print(list_newEventsOrder)
			}

			if (length(list_eventsOrder[[i]]) == 1){ # if only one element break
				break
			}

			if (isAnyOneNonSignifFromNextAll(list_eventsOrder, list_eventsOrder[[i]][j], i, signifs)){ # remove that element, and add to end.
				# add the jth one to a new next layer
				# print("here!")
				# print(list_eventsOrder[[i]][j])

				list_newEventsOrder <- rmElemAndAddSepElseNew(list_newEventsOrder, list_eventsOrder[[i]][j])
				# print("now list is")
				# print(list_newEventsOrder)

			}
		}

	}

	return (list_newEventsOrder)
}

checkAndAddEventAtEnd <- function(anEvent, list_newEventsOrder){
	isFound = FALSE
	for (i in 1:length(list_newEventsOrder)){
		if (anEvent %in% list_newEventsOrder){
			isFound = TRUE
		}
	}

	if (isFound == FALSE){
		list_newEventsOrder <- append(list_newEventsOrder, c(anEvent))
	}

	return(list_newEventsOrder)
}

rmElemAndAddSepElseNew <- function(list_newEventsOrder, theElem, theVec){
	# print("The eleme is ")
	# print(theElem)

	isDeleted = FALSE
	for (i in 1:length(list_newEventsOrder)){
		if (theElem %in% list_newEventsOrder[[i]]){
			idx = match(theElem, list_newEventsOrder[[i]])
			# print(" the idx is ")
			# print(idx)
			if (idx > 0){

				if (length (list_newEventsOrder[[i]]) > 1){
					list_newEventsOrder[[i]] <- list_newEventsOrder[[i]][-idx]
					isDeleted = TRUE
				}

				if (length(list_newEventsOrder[[i]]) == 0){
					print("Need to delete this here....?")
					list_newEventsOrder <- list_newEventsOrder[-i]
				}
			}

		}
	}
	if (isDeleted == TRUE){
		for (i in 1:length(list_newEventsOrder)){
			if (length(list_newEventsOrder[[i]]) == 0){
				list_newEventsOrder <- Filter(Negate(is.null), list_newEventsOrder)
			}
		}
		list_newEventsOrder <- append(list_newEventsOrder, c(theElem))
	}

	return (list_newEventsOrder)
}

rmElemAndAddSepElseNew_old <- function(list_newEventsOrder, theElem, theVec){
	# print("The eleme is ")
	# print(theElem)
	for (i in 1:length(list_newEventsOrder)){
		if (theElem %in% list_newEventsOrder[[i]]){
			idx = match(theElem, list_newEventsOrder[[i]])
			# print(" the idx is ")
			# print(idx)
			if (idx > 0){
				list_newEventsOrder[[i]] <- list_newEventsOrder[[i]][-idx]
			}

		}
	}

	list_newEventsOrder <- append(list_newEventsOrder, c(theElem))

	return (list_newEventsOrder)
}

isVecInList <- function(theVec, theList, elemToRm){

	for (subVec in theList){
		if (all(subVec==theVec)){
			return (TRUE)
		}
	}

	return (FALSE)
}

isAnyOneNonSignifFromNextAll <- function(list_eventsOrder, currEvent, currentLayer, signifs){
	if ((currentLayer + 1) >= length(list_eventsOrder)){
		return (FALSE)
	}

	for (i in seq((currentLayer + 1), length(list_eventsOrder))){

		isAny = isAnyNonSignifFromPrev(c(currEvent), list_eventsOrder[[i]], signifs)

		if (isAny == TRUE){
			return (TRUE)
		}
	}

	return (FALSE)
}

getEventsOrderInGraph <- function(reducted, net){
	vec_roots <- getRoots(reducted) # get root nodes
	vec_linCurrNodes <- vec_roots

	# list_currNodes <- as.list(vec_roots)

	vec_visited <- c()

	list_eventsOrder <- list()

	while(length(vec_linCurrNodes) > 0){

		# list_prevNodes = list_currNodes

		# print("vec_linCurrNodes")
		# print (vec_linCurrNodes)

		vec_newForVisit <- getNotVisitedBefore(vec_linCurrNodes, vec_visited)

		vec_newForVisit <- onlyPatron(vec_newForVisit, net)

		# print("to visit")
		# print (vec_newForVisit)

		if (length(vec_newForVisit) > 0){
			# print("to visit")
			# print (vec_newForVisit)

			list_eventsOrder <- addNodesToOrderList(vec_newForVisit, list_eventsOrder)
			vec_visited <- visitNodes(vec_newForVisit, vec_visited)
		}

		list_currNodes <- getAllInNextHop(net, vec_newForVisit, "out")
		vec_linCurrNodes <- convertToUniqueVec(list_currNodes)

		if (length(vec_linCurrNodes) == 0 && length(igraph::V(net)) != length(vec_visited)){
			vec_linCurrNodes <- setdiff(igraph::V(net), vec_visited)
		}
	}
	return (list_eventsOrder)
}

cols_clusPlotObjs$col_x0 <- 1
cols_clusPlotObjs$col_y0 <- 2
cols_clusPlotObjs$col_x1 <- 3
cols_clusPlotObjs$col_y1 <- 4
cols_clusPlotObjs$col_col <- 5

getNumOfCrossingAndDir <- function(mat_fiftyPoints, clusNum){

	totalNumOfCrossings <- 0
	list_dir = list()

	for (rowNum in 1:nrow(mat_fiftyPoints)){
		if (mat_fiftyPoints[rowNum, cols_matFifty$col_clus] == clusNum){
			totalNumOfCrossings = totalNumOfCrossings + 1
			list_dir = append(list_dir, mat_fiftyPoints[rowNum, cols_matFifty$col_dir])
		}
	}

	return (list(totalNumOfCrossings, list_dir))
}

genDistForEvent <- function(clusMat, dir, startTp, endTp, phosTh, dephosTh){
	mat_distribution <- matrix(ncol=4)

	for (profNum in 1:nrow(clusMat)){

		if ((dir == 1 && clusMat[profNum, startTp] < clusMat[profNum, endTp]) || (dir == -1 && clusMat[profNum, startTp] > clusMat[profNum, endTp])){
			# direction holds; do calculation, and save.
			# print(clusMat[profNum, startTp:endTp])

			crossPt <- calcCrossing_v3(clusMat[profNum, startTp:endTp], dir, (startTp -1), phosTh, dephosTh)

			mat_distribution <- addToMatrix(mat_distribution, profNum, crossPt[1], crossPt[2], dir)

		}
		else{
			mat_distribution <- addToMatrix(mat_distribution, profNum, NA, NA, dir)
		}
	}

	mat_distribution <- mat_distribution[-1,, drop=FALSE]
	return (mat_distribution)
}

getDistOfAllEvents_v2 <- function(mat_fiftyPoints, list_matrices, phosTh, dephosTh){
	list_distributions <- list()

	for (eventNum in 1:nrow(mat_fiftyPoints)){
		dist <- genDistForEvent(list_matrices[[mat_fiftyPoints[eventNum,cols_matFifty_v2$clus]]], mat_fiftyPoints[eventNum, cols_matFifty_v2$dir], mat_fiftyPoints[eventNum, cols_matFifty_v2$startTp], mat_fiftyPoints[eventNum, cols_matFifty_v2$endTp], phosTh, dephosTh)

		list_distributions[[length(list_distributions) + 1]] <-  dist
	}

	return(list_distributions)
}

isComparisonPossible <- function(list1, list2){
	list1 <- list1[!is.na(list1)]
	list2 <- list2[!is.na(list2)]

	# print(list1)

	if(all(abs(list1 - mean(list1)) < 0.0001) && all(abs(list2 - mean(list2)) < 0.0001) && abs(mean(list1) - mean(list2)) < 0.0001) {
		# print("false!!!)")
		return (FALSE)

	}

	return (TRUE)
}

performWilcoxonSignedRankTabular <- function(list_distributions){
	mat_pVal = matrix(nrow=length(list_distributions), ncol=length(list_distributions))
	mat_statistic = matrix(nrow=length(list_distributions), ncol=length(list_distributions))

	for (i in 1:length(list_distributions)){
		for (j in 1:length(list_distributions)){

			if (i != j){

				if (!isComparisonPossible(list_distributions[[i]][,cols_matFifty$col_x], list_distributions[[j]][,cols_matFifty$col_x])){ # add as pval = 1 (being the exact same from a uniform distribution).
					mat_pVal[i, j] <- 1
					mat_statistic[i, j] <- NA
				}
				else{ # do the comparison
					res <- stats::wilcox.test(list_distributions[[i]][,cols_matFifty$col_x], list_distributions[[j]][,cols_matFifty$col_x], paired=FALSE, conf.int=TRUE)

					# if (stats::wilcox.test(list_distributions[[i]][,cols_matFifty$col_x], list_distributions[[j]][,cols_matFifty$col_x], paired=FALSE, conf.int=TRUE)){
					#	print(paste("The case here ", i, j))

					#}
					# return (res)
					# print (res)
					mat_pVal[i, j] <- res$p.value
					mat_statistic[i, j] <- res$estimate
				}
			}
		}
	}

	list_pVal_stat <- list()
	list_pVal_stat[[1]] <- mat_pVal
	list_pVal_stat[[2]] <- mat_statistic
	return (list_pVal_stat)
}

performTtestsTabular <- function(list_distributions){
	mat_pVal = matrix(nrow=length(list_distributions), ncol=length(list_distributions))
	mat_statistic = matrix(nrow=length(list_distributions), ncol=length(list_distributions))

	for (i in 1:length(list_distributions)){
		for (j in 1:length(list_distributions)){

			if (i != j){
				if (!isComparisonPossible(list_distributions[[i]][,cols_matFifty$col_x], list_distributions[[j]][,cols_matFifty$col_x])){ # add as pval = 1 (being the exact same from a uniform distribution).
					mat_pVal[i, j] <- 1
					mat_statistic[i, j] <- NA
				}
				else{ # do the comparison
					res <- stats::t.test(list_distributions[[i]][,cols_matFifty$col_x], list_distributions[[j]][,cols_matFifty$col_x], paired=FALSE)
					mat_pVal[i, j] <- res$p.value
					mat_statistic[i, j] <- res$statistic
				}
			}
		}
	}

	list_pVal_stat <- list()
	list_pVal_stat[[1]] <- mat_pVal
	list_pVal_stat[[2]] <- mat_statistic
	return (list_pVal_stat)
}

################################ GRAPH_TRAVERSAL

getRoots <- function(reducted){
	vec_roots = c()
	for (colNum in 1:nrow(reducted)){

		isAllZeros = TRUE
		for (rowNum in 1:ncol(reducted)){
			if (reducted[rowNum, colNum] == 1){
				isAllZeros = FALSE
				break
			}
		}

		if (isAllZeros == TRUE){
			vec_roots <- append(vec_roots, colNum)
		}

	}

	return (vec_roots)
}

getNotVisitedBefore <- function(vec_currNodes, vec_visited){
	vec_newForVisit <- c()

	for (i in 1:length(vec_currNodes)){

		if (!(vec_currNodes[i] %in% vec_visited) && !(vec_currNodes[i] %in% vec_newForVisit)){
			vec_newForVisit <- c(vec_newForVisit, vec_currNodes)
		}
	}

	return (vec_newForVisit)
}

onlyPatron <- function(vec_nodes, net){

	# only add a node if it is not a child of anything else in this set.

	vec_parentsOnly <- c()

	for (i in 1:length(vec_nodes)){
		isChild = FALSE;

		vec_withoutCurr <- setdiff(vec_nodes, vec_nodes[i])

		if (length(vec_withoutCurr) > 0) {
			list_children <- getAllInNextHop(net, vec_withoutCurr, 'out')
			vec_children <- convertToUniqueVec(list_children)

			# print(vec_children)
			# print(vec_nodes[i])

			if (vec_nodes[i] %in% vec_children){
				isChild = TRUE
			}
			else { # make sure that this node is also not in the downstream lineage.
				if (length(vec_children) > 0){
					for (j in 1:length(vec_children)){
						if (igraph::are.connected(net, vec_children[j], vec_nodes[i]) == TRUE || igraph::vertex_connectivity(net, vec_children[j], vec_nodes[i]) == 1){
							isChild = TRUE
						}
					}
				}
			}

		}

		if (isChild == FALSE){
			vec_parentsOnly <- c(vec_parentsOnly, vec_nodes[i])
		}

	}

	return (vec_parentsOnly)
}

addNodesToOrderList <- function(vec_nodesInNewLayer, list_eventsOrder){
	pos <- length(list_eventsOrder) + 1
	list_eventsOrder[[pos]] <- vec_nodesInNewLayer
	return (list_eventsOrder)
}

visitNodes <- function(vec_newForVisit, vec_visited){

	vec_visited <- c(vec_visited, vec_newForVisit)

	return (vec_visited)
}

getAllInNextHop <- function(network, list_nodes, edgeDir){
	list_ <- igraph::adjacent_vertices(network, list_nodes, mode="out")

	return (list_)
}

convertToUniqueVec <- function(list_nodes){
	vec_nodes <- c()
	# print("lvl1")
	if (length(list_nodes) > 0){
		for (i in 1:length(list_nodes)){
			# print("lvl2")
			if (length(list_nodes[[i]]) > 0){
				for(j in 1:length(list_nodes[[i]])){
					# print(c(i,j), sep="\t")
					# print(list_nodes)
					if (! (list_nodes[[i]][j] %in% vec_nodes)){
						vec_nodes <- c(vec_nodes, list_nodes[[i]][j])
					}
				}
			}
		}
	}
	# print(list_nodes)
	return (vec_nodes)
}

###################### FOR PLOTTING
getXaxisLabels <- function(list_eventsOrder, mat_eventPoints, signifs, list_blocks, eventStart_x, xSigIncr, xNonSigIncr, yLabelInit, yLabelSpace){
	mat_xLabelsClus = matrix(ncol=4) #x0, y0, labelNames, color
	x0 = eventStart_x;
	eventsVisited = c()

	isFirst = TRUE

	for (i in 1:length(list_eventsOrder)){

		eventsVisited = c(eventsVisited, list_eventsOrder[[i]])

		if (i > 1 && isAnyNonSignifFromPrevAll(list_eventsOrder, i, signifs) == TRUE || isInSameBlock(list_eventsOrder[[i]], eventsVisited, list_blocks)){
			if (isFirst == FALSE){
				x0 = x0 + xNonSigIncr;
			}
			else{
				isFirst = FALSE
			}
		}
		else {
			if (isFirst == FALSE){
				x0 = x0 + xSigIncr;
			}
			else {
				isFirst = FALSE
			}

		}

		y0 = yLabelInit

		sortedEvents <- getSortedEventsByY(list_eventsOrder[[i]], mat_eventPoints)

		for (event in sortedEvents){

			# print(mat_eventPoints[event, cols_matFifty$col_dir])

			col = mat_eventPoints[event, cols_matFifty$col_dir]
			# col = color_events$up

			# if (mat_eventPoints[event, cols_matFifty$col_dir] != colors_orderedEvents$incr){
			# 	col = color_events$down
			# }

			rowVal = c(x0, y0, mat_eventPoints[event, cols_matFifty$col_clus], col)
			mat_xLabelsClus <- rbind(mat_xLabelsClus, rowVal)

			y0 = y0 - yLabelSpace;
		}

	}

	mat_xLabelsClus <- mat_xLabelsClus[-1,]
	return (mat_xLabelsClus)
}

getSortedEventsByY <- function(events, mat_eventPoints){
	theOrder <- rev(order(as.numeric(mat_eventPoints[events, cols_matFifty$col_y])))
	return(events[theOrder])
}

getGrayLines_v2 <- function(list_eventsOrder, mat_eventPoints, signifs){
	theGrayLines = matrix(ncol=4) # x0, x1, isSemiCircle, color

	for (i in 1:nrow(signifs)){
		for (j in i:ncol(signifs)){
			if (i != j){
				if (signifs[j, i] == FALSE){
					# if in same layer no need for line

					j_layer = getLayerNum(list_eventsOrder, j)
					i_layer = getLayerNum(list_eventsOrder, i)

					if (j_layer != i_layer){
						if (i_layer - 1 == j_layer || j_layer - 1 == i_layer){
							rowVal <- c(mat_eventPoints[i, cols_matFifty$col_x], mat_eventPoints[j, cols_matFifty$col_x], FALSE, "#808080")
						}
						else{
							rowVal <- c(mat_eventPoints[i, cols_matFifty$col_x], mat_eventPoints[j, cols_matFifty$col_x], TRUE, "#808080")
						}
						theGrayLines <- rbind(theGrayLines, rowVal)
					}
					# if in diff layer
						# if in consecutive layer, draw straight line

						# if not, draw semicircle.

					# draw a gray50 line between the i and j events

				}
			}
		}
	}
	theGrayLines <- theGrayLines[-1,]

	# print(theGrayLines)
	return (theGrayLines)
}

getLayerNum <- function(list_eventsOrder, event){
	for (i in 1:length(list_eventsOrder)){
		if (event %in% list_eventsOrder[[i]]){
			return (i)
		}
	}

	return (-1)
}

getGrayLines <- function(mat_eventPoints, signifs){
	theGrayLines = matrix(ncol=5)

	for (i in 1:nrow(signifs)){
		for (j in i:ncol(signifs)){
			if (i != j){
				if (signifs[j, i] == FALSE){
					# draw a gray50 line between the i and j events
					rowVal <-  c(mat_eventPoints[i, cols_matFifty$col_x], mat_eventPoints[i, cols_matFifty$col_y], mat_eventPoints[j, cols_matFifty$col_x], mat_eventPoints[j, cols_matFifty$col_y], "red")
					theGrayLines <- rbind(theGrayLines, rowVal)
				}
			}
		}
	}
	theGrayLines <- theGrayLines[-1,,drop=F]

	return (theGrayLines)
}

getRectPoints <- function(mat_eventPoints, list_blocks, rectStart, rectSpacingFmEvent){ #xStart, incr
	mat_rectPts = matrix(ncol=2) # [x0, x1]

	for (i in 1:length(list_blocks)){
		xVal <- max(as.numeric(mat_eventPoints[list_blocks[[i]], cols_matFifty$col_x])) + rectSpacingFmEvent

		if (i == 1){
			x0 <- rectStart
		}
		else{
			x0 <- mat_rectPts[nrow(mat_rectPts), cols_rectPoints$x1]
		}

		mat_rectPts <- rbind(mat_rectPts, c(x0, xVal))

	}
	mat_rectPts <- mat_rectPts[-1, ]
	return (mat_rectPts)
}

getRectBlock <- function(list_eventsOrder, signifs){
		list_blocks <- list()

		prevBlock <- c()
		currBlock <- list_eventsOrder[[1]]

		if(length(list_eventsOrder) > 1) {
			for (i in 2:length(list_eventsOrder)){

				isAny = isAnyNonSignifFromPrev(list_eventsOrder[[i]], currBlock, signifs)

				if (isAny == TRUE){
					currBlock <- c(currBlock, list_eventsOrder[[i]])

				}
				else{

					prevBlock <- c(prevBlock, currBlock)
					list_blocks[[length(list_blocks)+1]] <- currBlock
					currBlock <- list_eventsOrder[[i]]
					# rectPoints <- addToRectPoints(rectPoints, list_eventsOrder[[i]], mat_eventPoints)

					 # can improve further - i.e. take max of prevBlock listEvents x value (done)
				}

				# if (i == length(list_eventsOrder)){
			#		if (list_blocks[[length(list_blocks)]])
			#	}

				if (length(list_blocks) > 0 && length(prevBlock) > 0 && length(currBlock) > 0 && isAnyNonSignifFromPrev(currBlock, prevBlock, signifs) == TRUE){

					# print(length(list_blocks))
					# theCurrVec = list_blocks[[length(list_blocks)]]
					# theIdx = prevBlock == theCurrVec

					# print(theCurrVec)
					# print(prevBlock)

					# prevBlock <- prevBlock[!theIdx]
					prevBlock <- setdiff(prevBlock, list_blocks[[length(list_blocks)]])
					# prevBlock <- prevBlock[!prevBlock == joiningEvents]
					currBlock <- c(currBlock, list_blocks[[length(list_blocks)]])
					list_blocks <- list_blocks[-length(list_blocks)]

				}
			}

			# print(currBlock)
			if (length(list_blocks) > 0 && any(currBlock %in% list_blocks[[length(list_blocks)]]) == TRUE && all(currBlock %in% list_blocks[[length(list_blocks)]]) == FALSE ){
				list_blocks[[length(list_blocks)]] <- currBlock
			}
			else if (length(list_blocks) > 0 && all(currBlock %in% list_blocks[[length(list_blocks)]]) == FALSE){
				list_blocks[[length(list_blocks)+1]] <- currBlock
			}

		}
		# print("list blocks in getRectBlock")
		# print(list_blocks)
		# print("The list_blocks")
		# print(list_blocks)
		return (list_blocks)

}

addToRectPoints <- function(rectPoints, events, mat_eventPoints){
	# print(max(as.numeric(mat_eventPoints[events, cols_matFifty$col_x])))
	maxX <- max(as.numeric(mat_eventPoints[events, cols_matFifty$col_x]))

	rowVal = c()
	if (nrow(rectPoints) <= 1){
		rowVal <- c(-1, maxX-0.5)

	}
	else{
		rowVal <- c(rectPoints[nrow(rectPoints), 2], maxX-0.5)
	}
	# print("here")
	# print(rowVal)

	rectPoints <- rbind(rectPoints, rowVal)
	return (rectPoints)
}

makeDecrBarOpaque <- function(mat_clusConnLines, isCombined){

	if (isCombined == TRUE){
		for (i in 1:length(Color_multiomics)){
			idx <- mat_clusConnLines[,cols_clusPlotObjs$col_col] == Color_multiomics[[i]]$decr

			mat_clusConnLines[idx,cols_clusPlotObjs$col_col] = colors_orderedEvents$decr

			idx <- mat_clusConnLines[,cols_clusPlotObjs$col_col] == Color_multiomics[[i]]$incr

			mat_clusConnLines[idx,cols_clusPlotObjs$col_col] = Color_multiomics[[i]]$bar

		}
	}

	if (isCombined == FALSE){
		idx <- mat_clusConnLines[,cols_clusPlotObjs$col_col] == color_events$down

		mat_clusConnLines[idx,cols_clusPlotObjs$col_col] = colors_orderedEvents$decr

		idx <- mat_clusConnLines[,cols_clusPlotObjs$col_col] == color_events$up

		mat_clusConnLines[idx,cols_clusPlotObjs$col_col] = colors_orderedEvents$incr
	}

	return (mat_clusConnLines)
}

getClusConnLines <- function(mat_eventPoints, xStart, xEnd){
	mat_clusConnLines = matrix(ncol=5,nrow=nrow(mat_eventPoints))
	mat_basalConnLines = matrix(ncol=5,nrow=max(as.numeric(mat_eventPoints[,cols_matFifty$col_clus]))) # only need 1 for each cluster

	basalCounter = 1
	prevClus = -1

	for(i in 1:nrow(mat_eventPoints)){

		# adding start point
		mat_clusConnLines[i, cols_clusPlotObjs$col_x0] <- mat_eventPoints[i, cols_matFifty$col_x]
		mat_clusConnLines[i, cols_clusPlotObjs$col_y0] <- mat_eventPoints[i, cols_matFifty$col_y]

		# adding end y
		mat_clusConnLines[i, cols_clusPlotObjs$col_y1] <- mat_eventPoints[i, cols_matFifty$col_y]

		# adding end x
		mat_clusConnLines[i, cols_clusPlotObjs$col_x1] <- getTheEndX(mat_eventPoints, i, xEnd)

		# adding color
		mat_clusConnLines[i, cols_clusPlotObjs$col_col] <- mat_eventPoints[i, cols_matFifty$col_dir]

		## handling the adding of the basal
		if (prevClus == -1 ){
			# just store everything
			mat_basalConnLines[basalCounter, cols_clusPlotObjs$col_x0] = xStart;
			mat_basalConnLines[basalCounter, cols_clusPlotObjs$col_x1] = mat_clusConnLines[i, cols_clusPlotObjs$col_x0];
			mat_basalConnLines[basalCounter, cols_clusPlotObjs$col_y0] = mat_clusConnLines[i, cols_clusPlotObjs$col_y0];
			mat_basalConnLines[basalCounter, cols_clusPlotObjs$col_y1] = mat_clusConnLines[i, cols_clusPlotObjs$col_y1];

			mat_basalConnLines[basalCounter, cols_clusPlotObjs$col_col]  <- getOppColor(mat_eventPoints[i, cols_matFifty$col_dir])

			prevClus = mat_eventPoints[i,cols_matFifty$col_clus]
			# print(prevClus)
		}
		else if (prevClus == as.integer(mat_eventPoints[i,cols_matFifty$col_clus])){
			# update x if smaller (and then also color to be opposite)

			if (as.numeric(mat_clusConnLines[i, cols_clusPlotObjs$col_x0]) < as.numeric(mat_basalConnLines[basalCounter, cols_clusPlotObjs$col_x1]) ){
				mat_basalConnLines[basalCounter, cols_clusPlotObjs$col_x0] = mat_clusConnLines[i, cols_clusPlotObjs$col_x0];
				mat_basalConnLines[basalCounter, cols_clusPlotObjs$col_col]  <- getOppColor(mat_eventPoints[i, cols_matFifty$col_dir])
			}
		}
		else if (prevClus !=  mat_eventPoints[i,cols_matFifty$col_clus]){
			prevClus = mat_eventPoints[i,cols_matFifty$col_clus]
			basalCounter = basalCounter + 1
			# store everything again
			mat_basalConnLines[basalCounter, cols_clusPlotObjs$col_x0] = xStart;
			mat_basalConnLines[basalCounter, cols_clusPlotObjs$col_x1] = mat_clusConnLines[i, cols_clusPlotObjs$col_x0];
			mat_basalConnLines[basalCounter, cols_clusPlotObjs$col_y0] = mat_clusConnLines[i, cols_clusPlotObjs$col_y0];
			mat_basalConnLines[basalCounter, cols_clusPlotObjs$col_y1] = mat_clusConnLines[i, cols_clusPlotObjs$col_y1];

			mat_basalConnLines[basalCounter, cols_clusPlotObjs$col_col]  <- getOppColor(mat_eventPoints[i, cols_matFifty$col_dir])

		}

	}

	mat_clusConnLines = rbind(mat_clusConnLines, mat_basalConnLines)
	# print(mat_clusConnLines)
	# print(mat_eventPoints)
	return (mat_clusConnLines)

}

getOppColor <- function (thisCol){
	# omics colors
	if (thisCol == color_events$up){
		return (color_events$down)
	}
	else if (thisCol == color_events$down){
		return (color_events$up)
	}

	# multiomics colors
	for (i in 1:length(Color_multiomics)){
		if (thisCol == Color_multiomics[[i]]$incr){
			return (Color_multiomics[[i]]$decr)
		}
		else if (thisCol == Color_multiomics[[i]]$decr){
			return (Color_multiomics[[i]]$incr)
		}
	}

	return ("black")
}

getTheEndX <- function(mat_eventPoints, eventNum, defaultEndX){
	endX <- defaultEndX

	startX <- mat_eventPoints[eventNum, cols_matFifty$col_x]

	for (i in 1:nrow(mat_eventPoints)){
		if (i != eventNum){ # if not the current event
			if (mat_eventPoints[i, cols_matFifty$col_clus] == mat_eventPoints[eventNum, cols_matFifty$col_clus]){ # if same cluster
				if (as.numeric(mat_eventPoints[i, cols_matFifty$col_x]) > as.numeric(startX) && as.numeric(mat_eventPoints[i, cols_matFifty$col_x]) < as.numeric(endX)){ # if newX > startX && < endX; then update
					endX <- mat_eventPoints[i, cols_matFifty$col_x]
				}

			}
		}
	}

	return (endX)
}

getYaxisLabels <- function(mat_eventPoints){
	ord <- order(as.numeric(mat_eventPoints[,cols_matFifty$col_x]))
	clusNums <- as.numeric(mat_eventPoints[ord,1])
	clusNumLabels <- c()

	for(i in 1:length(clusNums)){
		if (! clusNums[[i]] %in% clusNumLabels){
			clusNumLabels <- append(clusNumLabels, clusNums[[i]])
		}
	}
	vec_labels =  as.vector(clusNumLabels)

	return (vec_labels)
}

getYIfClusPres <- function(currClusNum, mat_eventPoints){

	for (rowNum in 1:nrow(mat_eventPoints)){
		if (!is.na(mat_eventPoints[rowNum, cols_matFifty$col_clus]) && currClusNum == mat_eventPoints[rowNum, cols_matFifty$col_clus]){
			return (mat_eventPoints[rowNum, cols_matFifty$col_y])
		}
	}

	return (-1)
}

isAnyNonSignifFromPrevAll <- function(list_eventsOrder, currLayerNum, signifs){

	for (i in seq(1, (currLayerNum - 1))){
		isAny = isAnyNonSignifFromPrev(list_eventsOrder[[currLayerNum]], list_eventsOrder[[i]], signifs)
		if (isAny == TRUE){
			return (TRUE)
		}
	}

	return (FALSE)
}

getColorIfMultiOmics <- function(eventNum, mat_events, dir){
	datasetNum <- mat_events[eventNum, Col_events$combinedDatasetNum]

	if (is.na(datasetNum) && dir == 1){
		return(color_events$up)
	}
	else if (is.na(datasetNum) && dir == -1){
		return(color_events$down)
	}

	# otherwise its a multiomics data sets.
	if (dir == 1){
		return(Color_multiomics[[datasetNum]]$incr)
	}

	else if(dir == -1){
		return(Color_multiomics[[datasetNum]]$decr)
	}

}

getTheClusLines <- function(mat_fiftyPoints, list_eventsOrder, signifs, list_blocks, xStart, xIncrSig, xIncrNonSig, yDecr){ #xStart, yStart, y_decr, x_incr_sig, x_incr_nonSig

	mat_eventPoints <- matrix(nrow=nrow(mat_fiftyPoints), ncol=4)

	x <- xStart
	y <- max(mat_fiftyPoints[,1])

	eventsVisited = c()

	for (i in 1:length(list_eventsOrder)){
		for (j in 1:length(list_eventsOrder[[i]])){
			eventNum <- list_eventsOrder[[i]][j]

			# adding y coordinate
			prevY <- getYIfClusPres(mat_fiftyPoints[eventNum, cols_matFifty$col_clus], mat_eventPoints)
			if (prevY == -1){
				mat_eventPoints[eventNum, cols_matFifty$col_y] <- y
				y <- y - yDecr
			}
			else{
				mat_eventPoints[eventNum, cols_matFifty$col_y] <- prevY
			}

			# assigning color

			# if (mat_fiftyPoints[eventNum, cols_matFifty$col_dir] == 1){
			theColor <- getColorIfMultiOmics(eventNum, mat_fiftyPoints, mat_fiftyPoints[eventNum, cols_matFifty$col_dir])

			mat_eventPoints[eventNum, cols_matFifty$col_dir] = theColor
			# }
			#else{
			#	theColor <- getColorIfMultiOmics(eventNum, mat_fiftyPoints, -1)
			#	mat_eventPoints[eventNum, cols_matFifty$col_dir] = theColor
			#}

			# assigning cluster number
			mat_eventPoints[eventNum, cols_matFifty$col_clus] <- mat_fiftyPoints[eventNum, cols_matFifty$col_clus]

			eventsVisited <- c(eventsVisited, eventNum)
		}

		# adding x coordinate
		if (i == 1){
			#print("over here..?")
			mat_eventPoints <- addXForAll(list_eventsOrder[[i]], mat_eventPoints, x)
		}
		else{
			# if (isAnyNonSignifFromPrev(list_eventsOrder[[i]], list_eventsOrder[[i-1]], signifs) == TRUE){ # place events closer together
			if (isAnyNonSignifFromPrevAll(list_eventsOrder, i, signifs) == TRUE || isInSameBlock(list_eventsOrder[[i]], eventsVisited, list_blocks)){
				x <- x + xIncrNonSig
			}
			else {
				x <- x + xIncrSig
			}

			mat_eventPoints <- addXForAll(list_eventsOrder[[i]], mat_eventPoints, x)

		}
	}

	# print("The mat event points:")
	# print (mat_eventPoints)
	return (mat_eventPoints)

}

isInSameBlock <- function(events, eventsVisited, list_blocks){

	# 1. find block.
	# 2. exclude events from block, and from eventsVisited.
	# 3. See if others in block, which are also visited.
	# if so, return true, else return false.

	blockNum <- findBlock(events[1], list_blocks)

	eventsInBlock <-  as.vector(list_blocks[[blockNum]])
	otherEventsInBlock <- setdiff(eventsInBlock, events)

	otherEventsVisited <- setdiff(eventsVisited, events)

	if (length(otherEventsInBlock) > 0 && length(otherEventsVisited) > 0 && (any(otherEventsVisited %in% otherEventsInBlock) || any(otherEventsInBlock %in% otherEventsVisited))) {
		return (TRUE)
	}

	return (FALSE)
}

findBlock <- function(event, list_blocks){
	i = -1
	for (i in 1:length(list_blocks)){
		if (event %in% list_blocks[[i]]){
			return (i)
		}
	}

	return (i)
}

isAnyNonSignifFromPrev <- function(vec_curr, vec_prev, signifs){

	for (i in 1:length(vec_curr)){
		for (j in 1:length(vec_prev)){

			# print(paste(i, j, length(vec_curr), length(vec_prev), signifs[vec_curr[i], vec_prev[j]]))

			if (signifs[vec_curr[i], vec_prev[j]] == FALSE){
				return (TRUE)
			}

		}
	}

	return (FALSE)
}

addXForAll <- function(vec_curr, mat_eventPoints, x){

	for (i in 1:length(vec_curr)){
		mat_eventPoints[vec_curr[i], cols_matFifty$col_x] <- x
	}

	return (mat_eventPoints)
}


################################################################################
#        1         2         3         4         5          6        7         8
#2345678901234567890123456789012345678901234567890123456789012345678901234567890
################################################################################
# PARAMETER FUNCTION
# functions used in estimation of parameters for warmouth model
#-------------------------------------------------------------------------------
# Growth functions

# Function to est growth based on VB relationship
# can use form with t0 or form with L0
# use of either form based on specification of t0 or L0
# can only specify 1 and other must be NA
VB.growth <- function(t, Linf, K, t0 = NA, L0 = NA, names = FALSE) {

    if(is.na(L0[1])){
        L <- Linf * (1 - exp(-K * (t - t0)))  # t0 form
    } else if (is.na(t0[1])) {
        L <- Linf - (Linf - L0) * exp(-K * t) # L0 form
    }
    if(names == TRUE) names(L) <- paste0("L", t)
    L
}

# function to est age from VB growth function
# can use form with t0 or form with L0
# use of either form based on specification of t0 or L0
# can only specify 1 and other must be NA
agebyVB <- function(L, Linf, K, t0 = NA, L0 = NA){

    if(is.na(L0[1])){
        t <- -1 / K * log( 1 - L / Linf) + t0    # t0 form
    } else if (is.na(t0[1])) {
        t <- log(-((L - Linf) / (Linf - L0)))/-K # L0 form
    }
    t
}

#-------------------------------------------------------------------------------
# Estimate population growth rate and predict confidence/prediction interval
# Based on data and fit from Randall et al (1995) and adjustemnt to relationship
# from Randall and Minns (2000)

r_max_pred <- function(w,         # prediction weight
                       h,         # prediction habitat (1 = lake; 2 = river)
                       interval){ # confidence or prediction interval

    # Data from Randall et al. (1995)
    data <- as.data.frame(list(
        Production = c(30.3, 3.8, 2.0, 228.5, 15.5, 4.7, 21.5, 30.8, 5.7, 85.7, 8.6,
                       270.0, 596.0, 198.0, 140.0, 100.0,  74.0,  52.0, 30.0, 174.5,
                       280.3, 243.5, 226.4, 375.4, 379.1, 217.6, 500.6, 297.2, 110.3,
                       128.8, 161.4, 83.4, 807.2, 228.0, 329.0, 168.7, 57.2, 147.3,
                       54.8, 76.5, 30.7, 35.3, 261.5, 98.3, 39.3, 33.3, 47.5, 26.1,
                       31.2, 53.0, 456.2, 80.3, 29.4),
        biomass = c(73.0, 7.3, 9.2, 372.2, 32.2, 11.2, 28.9, 11.6, 24.4, 277.8, 29.5,
                    161.2, 198.0,  95.0,  75.0,  59.0,  51.0, 129.0, 62.0, 124.4,
                    274.8, 228.4, 149.9, 376.0, 278.7, 127.5, 217.5, 187.2, 111.2,
                    241.0, 307.5, 50.2, 310.5, 142.5, 86.6, 45.6, 10.8, 40.9, 49.8,
                    42.5, 38.4, 37.4, 173.0, 71.0, 30.5, 21.3, 38.5, 21.5, 27.1, 36.0,
                    233.4, 80.2, 15.4),
        weight = c(18.0246914, 3.5540409, 25.8426966, 6.6786291, 24.2105263, 37.8378378,
                   17.8947368, 5.7058534, 219.8198198, 13.6712598, 64.2701525, 1.24,
                   0.2544987, 1.4843750, 3.0, 2.4583333, 5.6666667, 18.4285714, 15.5,
                   2.4777915, 2.3869707, 1.4815487, 0.7197976, 3.7586469, 1.3582997,
                   2.0675564, 1.6603053, 0.6602313, 2.8333376, 13.9872316, 6.1428743,
                   2.1183222, 9.3047648, 4.2462529, 4.4194948, 0.4892704, 1.6410880,
                   0.8880686, 7.9935795,   0.9173916, 5.9222702, 10.0700054, 0.8397202,
                   1.3179144, 1.9223497, 2.1759117, 3.3804548, 2.6022755, 3.7215051,
                   2.7067669, 5.1344098, 12.0819524, 0.6273679),
        habitat = c(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
                    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
                    1, 1, 1, 1, 1, 1, 1, 1, 1)
    ))

    # fit regression model
    mod = lm(log10(Production/biomass) ~ log10(weight) + habitat, data = data)

    # x prediction values
    pred.data = data.frame(weight = w, habitat = h)

    # make prediction with desired interval estiamte
    rmax = (predict(mod, pred.data, interval = interval))

    # retransform and multiply by 2 (Randall and Minns 2000)
    10^rmax*2
}

#-------------------------------------------------------------------------------
# OTHER

# function to calc geometric mean between sequenital values in a vector
geom_mean_seq <- function(vec) {
    n = length(vec)
    sqrt(vec[2:n]*vec[1:(n-1)])
}

# geometric mean of a vector
geom_mean <- function(vec, na.rm = FALSE) {
    exp(mean(log(vec), na.rm = na.rm))
}

#-------------------------------------------------------------------------------
# For parameterizing projection matrix from expression

pmx_eval = function(mx, vars){
  matrix(sapply(mx, eval, vars),  sqrt(length(mx)), sqrt(length(mx)), byrow = T)
}

#===============================================================================
# Distribution functions

# Function to estimate shape parameters of the beta distribution
# from the defined mean and variance of data
beta_val <- function(mean, var){

  a <- mean * ((mean * (1 - mean)) / var - 1)
  b <- (1 - mean) * ((mean * (1 - mean)) / var - 1)

  list(a = a, b = b)
}

# Function to estimate shape parameters of the streched beta distribution
# from the defined mean and variance of data
# Streched beta is a beta dist'n scaled to outside (0,1) limits
# shape parameters give regulare beat dist'n which must be scaled with
# (max - min) + min when generating values with rbeta
beta_stretch_val <- function(mean, var = NA, sd = NA){

  if(is.na(var[1])) var = sd^2    # calc variance if not given
  if(is.na(sd[1])) sd = sqrt(var) # calc sd if not given

  # estimate mins and max of distribution
  min = qnorm(1e-6, mean, sd = sd)
  max = qnorm(1 - 1e-6, mean, sd = sd)

  # Scale means and variance to between 0 and 1
  mean.b <- (mean - min) / (max - min)
  var.b <- var * (1 / (max - min))^2

  # estimate beta dist'n shape parameters
  a <- mean.b * ((mean.b * (1 - mean.b)) / var.b - 1)
  b <- (1 - mean.b) * ((mean.b * (1 - mean.b)) / var.b - 1)

  # output
  list(a = a, b = b, min = min, max = max)
}

#===============================================================================
# Initial population size vector
# est. pop vector from desire adult population size

init_pop <- function(mx,   # Population Matrix
                     Nt,   # desire adult density
                     p.rep # maturity vector - 1: mature; 0: Juvenle; length Tmax + 1
)
{

  SS <- stable.stage(mx)                           # Stable stage distn
  Na <- SS * p.rep / sum(SS * p.rep) * Nt          # Calc adult pop
  Nj <- SS * (1-p.rep)/ sum(SS * p.rep) * Nt       # Calc juv pop
  N <- Nj + Na
  N
}

#===============================================================================

# Function to project population size including density dependence effects
Projection <- function(mx,          # projection matrix expression
                       data,        # life history data
                       lh_mean,     # mean life history data
                       N0,          # initial age1+ population size pop size
                       years,       # years to run simulation
                       p.cat,       # Probability of castastrophe
                       density_dependence = FALSE, # Boolean; if density-depedence included
                       demographic_stoch = FALSE,  # Boolean: if dempgraphic stochasticity included
                       allee = FALSE               # Boolean: if allee effects included
){
  #-----------------------------------------------------------------------------

  # needs popbio library
  require(popbio)

  # Castastrophes
  Catastrophe = sample(c(1,0), years, replace = TRUE,
                       prob = c(p.cat / data$gen.time, 1 - p.cat  / data$gen.time))

  # effect on castastrophe on pop size (percent recuction) - scaled Beta dist'n fit from Reed et al. (2003)
  e.cat <- sapply(Catastrophe, function(x) {
    ifelse(x == 0, NA, rbeta(1, shape1 = 0.762, shape2 = 1.5) * (1 - .5) + .5)
  })

  # Initialize parameters
  lh <- lh_mean

  fs <- f_rand(with(lh, c(f1, f2, f3, f4)), sd = data$f.logsd)
  lh$f1 <- fs[1]; lh$f2 <- fs[2]; lh$f3 <- fs[3]; lh$f4 <- fs[4];

  ss <- s_rand(with(lh, c(s0, s1, s2, s3)), cv = data$M.cv)
  lh$s0 <- ss$ss[1]; lh$s1 <- ss$ss[2]; lh$s2 <- ss$ss[3]; lh$s3 <- ss$ss[4];

  # Population matrix
  A <- pmx_eval(mx, lh)

  # Initial population structure
  N <- as.vector(stable.stage(A) * N0 )
  #N <- as.vector(A %*% init_pop(A, Na, data$p.rep) )

  # initialize output vectors
  Nvec <- rep(NA, years + 1); Nvec[1] <- sum(N) # * data$p.rep)        # Adult pop vector
  Evec <- rep(NA, years + 1);
  s0s <- NULL       # density dependent parameters
  Ns <- list(N)              # age-specific annual population size
  lambdas <- rep(NA, years)  # population growth rate

  # stochastic parameters
  fts <- list(fs)
  sts <- list(ss$ss)

  #-----------------------------------------------------------------------------
  # loop through years
  for(t in 1:years){

    #---------------------------------------------------------------------------
    # Density Dependence

    # Allee Effects
    if(allee) {

      lh$a.f <- sum(N*data$p.rep) ^ 2 / (data$a ^ 2 + sum(N*data$p.rep) ^ 2)
      if(is.finite(lh$a.f) == F) lh$a.f = 0

    }

    # Survival Density-Dependence
    Et <- E_est(lh, N) # Egg count
    Evec[t] <- Et      # save egg count in vector
    if(density_dependence) {

      #s0.d <- data$s0.max * exp(-data$b * Et)
      s0.d <- data$s0.max / (1 + data$b * Et)
      if(s0.d < 1e-300) s0.d <- 1e-300

      # Mean survival rate
      lh_mean$s0 <- s0.d
    }

    # Demographic Stochasticity
    # simlulates random variation in vital rates when pop size is small.
    # make N random drawns for each age class for vital rates. as N increase the
    # draw will be closer to the mean, samller Ns will be more stochastic
    if(demographic_stoch) {
      # Survival rate
      # Binomial distribution - Sum N drawn and divide by N to give mean Survival
      lh$s0 <- lh_mean$s0
      lh$s1 <- lh$s2 <- lh$s3 <- ifelse(sum(N) > 1 & sum(N) < 500,
                                        sum(rbinom(sum(N), 1, lh_mean$s1))/sum(N),
                                        lh_mean$s1)

      # Fecundity
      # Poison dist. - take mean of N/2 (females) draws to give mean fecundity
      lh$f1 <- lh$f2 <- lh$f3 <- lh$f4 <- ifelse(sum(N*data$p.rep) > 2 & sum(N*data$p.rep) < 500,
                                                 mean(rpois(sum(N*data$p.rep)/2, lh_mean$f2)),
                                                 lh_mean$f2)
    }else {
      lh <- lh_mean
    }

    #---------------------------------------------------------------------------
    # Stochasticity

    # Fecundity
    fs <- f_rand(with(lh, c(f1, f2, f3, f4)), sd = data$f.logsd)
    lh$f1 <- fs[1]; lh$f2 <- fs[2]; lh$f3 <- fs[3]; lh$f4 <- fs[4];

    # Survival
    ss <- s_rand(with(lh, c(s0, s1, s2, s3)), cv = data$M.cv)
    lh$s0 <- ss$ss[1]; lh$s1 <- ss$ss[2]; lh$s2 <- ss$ss[3]; lh$s3 <- ss$ss[4];

    #---------------------------------------------------------------------------
    # Population

    # Projection matrix
    A <- pmx_eval(mx, lh)

    # project the population 1 year ahead.
    if(Catastrophe[t] == 1){
      N <- N * (1 - e.cat[t])
    } else {
      N <- as.vector(A %*% N )
    }

    Nvec[t + 1] <- sum(N)         # Number of mature fish in pop
    s0s <- c(s0s, lh_mean$s0)

    Ns[t + 1] <- list(N)

    lambdas[t] <- Nvec[t + 1]/Nvec[t] # pop growth rate

    fts[t + 1] <- list(fs)
    sts[t + 1] <- list(ss$ss)

  }

  #-----------------------------------------------------------------------------
  # output
  list(pop = as.data.frame(list(year = 0:years,
                                N = Nvec,
                                E = Evec,
                                s0 = c(s0s, NA))),

       N = do.call(rbind, Ns),

       lambdas = lambdas,
       vars = list(
         ft = do.call(rbind, fts),
         st = do.call(rbind, sts)
       ),
       Cat = as.data.frame(list(
         Cat = Catastrophe,
         e.cat = e.cat
       ))

  )
}

#===============================================================================
# Repatriation

repatriation <- function(mx,                # projection matrix expression
                         data,              # life history data
                         lh_mean,           # mean life history data
                         Ninit = 0,         # initial adult population size pop size
                         Nt.vec,            # transfered population as vector of ages 0:tmax
                         s.t,               # survvial rate during translocation
                         t.time,            # when stocking takes place ="pre.spawn" OR "post.spawn"
                         stock.month,       # num. months post spawned stocking takes place
                         stock.max,         # number of stocking events to take place
                         years,             # years to run simulation
                         p.cat              # Probability of castastrophe
){
  #-----------------------------------------------------------------------------

  # needs popbio library
  require(popbio)

  # Castastrophes
  Catastrophe = sample(c(1,0), years, replace = TRUE,
                       prob = c(p.cat / data$gen.time, 1 - p.cat  / data$gen.time))

  # effect on castastrophe on pop size (percent recuction) - scaled Beta dist'n fit from Reed et al. (2003)
  e.cat <- sapply(Catastrophe, function(x) {
    ifelse(x == 0, NA, rbeta(1, shape1 = 0.762, shape2 = 1.5) * (1 - .5) + .5)
  })

  # Initialize parameters
  lh <- lh_mean

  fs <- f_rand(with(lh_mean , c(f1, f2, f3, f4)), sd = data$f.logsd)
  lh$f1 <- fs[1]; lh$f2 <- fs[2]; lh$f3 <- fs[3]; lh$f4 <- fs[4];

  ss <- s_rand(with(lh_mean , c(s0, s1, s2, s3)), cv = data$M.cv)
  lh$s0 <- ss$ss[1]; lh$s1 <- ss$ss[2]; lh$s2 <- ss$ss[3]; lh$s3 <- ss$ss[4];

  # Population matrix
  A <- pmx_eval(mx, lh)

  # Initial population structure
  N <- as.vector(stable.stage(A) * Ninit )

  # initialize output vectors
  Nvec <- rep(NA, years);    #Adult pop vector
  Ns <- list(N)              # age-specific annual population size

  # stock counter
  stock.count <- 0

  #-----------------------------------------------------------------------------
  # loop through years
  for(t in 1:years){

    # stocking pre spawning
    if(t.time == "pre.spawn" & stock.count < stock.max){
      N <- N + Nt.vec * s.t
      stock.count <- stock.count + 1
    }

    #---------------------------------------------------------------------------
    # Density Dependence

    # Allee
    lh$a.f <- sum(N*data$p.rep) ^ 2 / (data$a ^ 2 + sum(N*data$p.rep) ^ 2)
    if(is.finite(lh$a.f) == F) lh$a.f = 0

    # SURVIVAL
    Et <- E_est(lh, N)

    #s0.d <- data$s0.max * exp(-data$b * Et)
    s0.d <- data$s0.max / (1 + data$b * Et)
    if(s0.d < 1e-300) s0.d <- 1e-300

    # Mean survival rate
    lh$s0 <- s0.d

    #---------------------------------------------------------------------------
    # Demographic Stochasticity
    # simlulates random variation in vital rates when pop size is small.
    # make N random drawns for each age class for vital rates. as N increase the
    # draw will be closer to the mean, samller Ns will be more stochastic

    # Survival rate
    # Binomial distribution - Sum N drawn and divide by N to give mean Survival
    lh$s1 <- lh$s2 <- lh$s3 <- ifelse(sum(N) > 1 & sum(N) < 500,
                                      sum(rbinom(sum(N), 1, lh_mean$s1))/sum(N),
                                      lh_mean$s1)

    # Fecundity
    # Poison dist. - take mean of N/2 (females) draws to give mean fecundity
    lh$f1 <- lh$f2 <- lh$f3 <- lh$f4 <- ifelse(sum(N*data$p.rep) > 2 & sum(N*data$p.rep) < 500,
                                               mean(rpois(sum(N*data$p.rep)/2, lh_mean$f2)),
                                               lh_mean$f2)

    #---------------------------------------------------------------------------
    # Stochasticity

    # Fecundity
    fs <- f_rand(with(lh, c(f1, f2, f3, f4)), sd = data$f.logsd)
    lh$f1 <- fs[1]; lh$f2 <- fs[2]; lh$f3 <- fs[3]; lh$f4 <- fs[4];

    # Survival
    ss <- s_rand(means = with(lh, c(s0, s1, s2, s3)), cv = data$M.cv, X = NA)
    lh$s0 <- ss$ss[1]; lh$s1 <- ss$ss[2]; lh$s2 <- ss$ss[3]; lh$s3 <- ss$ss[4];

    #---------------------------------------------------------------------------
    # Population

    # Projection matrix
    A <- pmx_eval(mx, lh)

    # project the population 1 year ahead.
    if(Catastrophe[t] == 1){
      N <- N * (1 - e.cat[t])
    } else {
      N <- as.vector(A %*% N )
    }

    # Stocking - post spawning
    # Stocked fish must survival the year before spawning - included stochasticity
    if(t.time == "post.spawn" & stock.count < stock.max){

      # Demographic stochasticity
      lh.s <- lh
      lh.s$s1 <- lh.s$s2 <- lh.s$s3 <- ifelse(sum(Nt.vec) > 1 & sum(Nt.vec) < 500,
                                        sum(rbinom(sum(Nt.vec), 1, lh_mean$s1))/sum(Nt.vec),
                                        lh_mean$s1)

      # Environmental stochasticity - same residuals as in matrix (X = ss$X)
      ss <- s_rand(with(lh.s, c(s0, s1, s2, s3)), cv = data$M.cv, X = ss$X)
      lh.s$s0 <- ss$ss[1]; lh.s$s1 <- ss$ss[2]; lh.s$s2 <- ss$ss[3]; lh.s$s3 <- ss$ss[4];

      # add stocked fish to pop vector - include mortality scaled to time remaiing in year
      N <- N + Nt.vec * s.t * with(lh.s, c(s1, s2, s3, 0)) ^ ((12 - stock.month)/12)

      # increment stocking counter
      stock.count <- stock.count + 1
    }

    # outputs
    Nvec[t] <- sum(N)         # Number of mature fish in pop

    Ns[t] <- list(N)          # Pop vector for output

  }
  #-----------------------------------------------------------------------------
  # output
  list(pop = as.data.frame(list(year = 1:years,
                                N = Nvec)),

       N = do.call(rbind, Ns)
  )
}

#===============================================================================
# Source Population Harm

# Function to project population size including density dependence effects
source_harm <- function(mx,          # projection matrix expression
                        data,        # life history data
                        lh_mean,     # mean life history data
                        Ninit ,      # initial adult population size pop size
                        Nt.vec,      # transfered population as vector of ages 0:tmax
                        t.time,      # when stocking takes place ="pre.spawn" OR "post.spawn"
                        stock.max,   # number of stocking events to take place
                        years,       # years to run simulation
                        p.cat        # Probability of castastrophe "Length" and or "Survival"
){
  #-----------------------------------------------------------------------------

  # needs popbio library
  require(popbio)

  # Castastrophes
  Catastrophe = sample(c(1,0), years, replace = TRUE,
                       prob = c(p.cat / data$gen.time, 1 - p.cat  / data$gen.time))

  # effect on castastrophe on pop size (percent recuction) - scaled Beta dist'n fit from Reed et al. (2003)
  e.cat <- sapply(Catastrophe, function(x) {
    ifelse(x == 0, NA, rbeta(1, shape1 = 0.762, shape2 = 1.5) * (1 - .5) + .5)
  })

  # Initialize parameters
  lh <- lh_mean

  fs <- f_rand(with(lh_mean , c(f1, f2, f3, f4)), sd = data$f.logsd)
  lh$f1 <- fs[1]; lh$f2 <- fs[2]; lh$f3 <- fs[3]; lh$f4 <- fs[4];

  ss <- s_rand(with(lh_mean , c(s0, s1, s2, s3)), cv = data$M.cv)
  lh$s0 <- ss$ss[1]; lh$s1 <- ss$ss[2]; lh$s2 <- ss$ss[3]; lh$s3 <- ss$ss[4];

  # Population matrix
  A <- pmx_eval(mx, lh)

  # Initial population structure
  N <- as.vector(stable.stage(A) * Ninit)

  # initialize output vectors
  Nvec <- rep(NA, years + 1); Nvec[1] <- sum(N)# * data$p.rep)        # Adult pop vector
  Ns <- list(N)              # age-specific annual population size

  # stock counter
  stock.count <- 0

  #-----------------------------------------------------------------------------
  # loop through years
  for(t in 1:years){

    # stocking pre spawning
    if(t.time == "pre.spawn" & stock.count < stock.max){
      N <- N - Nt.vec
      N <- ifelse(N > 1, N, 0)
      stock.count <- stock.count + 1
    }

    #---------------------------------------------------------------------------
    # Density Dependence

    # Allee Effects
    lh$a.f <- sum(N*data$p.rep) ^ 2 / (data$a ^ 2 + sum(N*data$p.rep) ^ 2)
    if(is.finite(lh$a.f) == F) lh$a.f = 0

    # Survival Density-Dependence
    Et <- E_est(lh, N) # Egg count
    s0.d <- data$s0.max / (1 + data$b * Et)
    if(s0.d < 1e-300) s0.d <- 1e-300

    # Mean survival rate
    lh_mean$s0 <- s0.d

    # Demographic Stochasticity
    # simlulates random variation in vital rates when pop size is small.
    # make N random drawns for each age class for vital rates. as N increase the
    # draw will be closer to the mean, samller Ns will be more stochastic

    # Survival rate
    # Binomial distribution - Sum N drawn and divide by N to give mean Survival
    lh$s0 <- lh_mean$s0
    lh$s1 <- lh$s2 <- lh$s3 <- ifelse(sum(N) > 1 & sum(N) < 500,
                                      sum(rbinom(sum(N), 1, lh_mean$s1))/sum(N),
                                      lh_mean$s1)

    # Fecundity
    # Poison dist. - take mean of N/2 (females) draws to give mean fecundity
    lh$f1 <- lh$f2 <- lh$f3 <- lh$f4 <- ifelse(sum(N*data$p.rep) > 2 & sum(N*data$p.rep) < 500,
                                               mean(rpois(sum(N*data$p.rep)/2, lh_mean$f2)),
                                               lh_mean$f2)

    #---------------------------------------------------------------------------
    # Stochasticity

    # Fecundity
    fs <- f_rand(with(lh_mean , c(f1, f2, f3, f4)), sd = data$f.logsd)
    lh$f1 <- fs[1]; lh$f2 <- fs[2]; lh$f3 <- fs[3]; lh$f4 <- fs[4];

    # Survival
    ss <- s_rand(with(lh_mean , c(s0, s1, s2, s3)), cv = data$M.cv)
    lh$s0 <- ss$ss[1]; lh$s1 <- ss$ss[2]; lh$s2 <- ss$ss[3]; lh$s3 <- ss$ss[4];

    #---------------------------------------------------------------------------
    # Population

    # Projection matrix
    A <- pmx_eval(mx, lh)

    # project the population 1 year ahead.
    if(Catastrophe[t] == 1){
      N <- N * (1 - e.cat[t])
    } else {
      N <- as.vector(A %*% N )
    }
    N <- ifelse(N > 1, N, 0)

    #output
    Nvec[t + 1] <- sum(N) # Number of age-1+ fish in pop
    Ns[t + 1] <- list(N)  # pop vector

    # Stocking - post spawning
    if(t.time == "post.spawn" & stock.count < stock.max){
      N <- N - Nt.vec
      N <- ifelse(N > 1, N, 0)
      stock.count <- stock.count + 1
    }

  }

  #-----------------------------------------------------------------------------
  # output
  list(pop = as.data.frame(list(year = 0:years,
                                N = Nvec
                                )),

       N = do.call(rbind, Ns)
  )
}

#===============================================================================


DL = function(y,sW){
k          = length(y)
w = sW^2
sum.w      = sum(w)
mu.hat     = sum(y*w)/sum.w
W          = sum.w - (sum(w^2)/sum.w)
Q          = sum(w*(y-mu.hat)^2)
typS       = sum(w*(k-1))/(sum.w^2 - sum(w^2))
Qp         = 1-pchisq(Q,k-1)

tausq.hat  = max(0,(Q-(k-1))/W)
w.tauhat   = 1/((1/w)+tausq.hat)

return(tausq.hat)

}

eff <- c(0.1, 0.2, 0.21, 0.5, 0.3)
se <- c(0.02, 0.3, 0.03, 0.07, 0.1)

a <- rma(yi=eff, sei=se, method="FE")
b <- rma(yi=eff, sei=se, method="DL")

DL = function(y=logHR, s = se.logHR)
{
	k            = length(y)
	w            = 1/s^2
	sum.w        = sum(w)
	mu.hat       = sum(y*w)/sum.w
	W            = sum.w - (sum(w^2)/sum.w)
	Q            = sum(w*(y-mu.hat)^2)
	typS         = sum(w*(k-1))/(sum.w^2 - sum(w^2))
	Qp           = 1-pchisq(Q,k-1)

	# fixed effects estimate

	Var.muhat    = 1/sum(w)
	muFE.CI      = mu.hat + c(-1.96,1.96)*sqrt(Var.muhat)
	Zfe          = (mu.hat)/sqrt(Var.muhat)
	FEp          = 2*(1 - pnorm(abs(Zfe)))

	# DL random effects estimate

	tausq.hat    = max(0,(Q-(k-1))/W)
	w.tauhat     = 1/(s^2+tausq.hat)
	mu.tau.hat   = sum(w.tauhat*y)/sum(w.tauhat)
	Var.mutauhat = 1/sum(w.tauhat)
	muRE.CI      = mu.tau.hat + c(-1.96,1.96)*sqrt(Var.mutauhat)
	Isq          = tausq.hat/(tausq.hat+typS)
	Zre          = (mu.tau.hat)/sqrt(Var.mutauhat)
	REp          = 2*(1 - pnorm(abs(Zre)))

	sum.wre      = sum(w.tauhat)

	summary      = c(k,Q,Qp,(100*Isq),mu.hat,muFE.CI,
	                 FEp,mu.tau.hat,muRE.CI,REp)

	out = list(
		fe=mu.hat,
		fe_se = sqrt(Var.muhat),
		re=mu.tau.hat,
		re_se = sqrt(Var.mutauhat)
	)

	return(out)
}

DL(eff, se)

rs1100405       t       c       -0.0178 0.0132  0.177   +--     0.0     0.388   2       0.8237  0.01514304      0.02311397      0.9879

CHR	POS	SNP	N	EFFECT_ALLELE	NON_EFFECT_ALLELE	EFFECT_ALLELE_FREQ	BETA	SE	r2	r2hat	P_VAL
7	43517495	rs1100405	1467	4	2	0.342195	0.00101	0.03945	4.47e-07	0.996	0.9798

CHR POS SNP STRAND EFFECT_ALLELE NON_EFFECT_ALLELE FREQ_EFFECT N BETA SE r2 PVALUE r2hat
7 43517495 rs1100405 + C T 0.665 1233 0.023 0.016 0.00152 0.1674 0.9958

SNP	CHR	POS	Rsq	AL1	AL2	FREQ1	TRAIT	EFFECT	SE	H2	LOD	PVALUE
rs1100405	7	43517495	0.9986	C	T	0.666	glucose_ND	0.011	0.029	0.008	0.032	0.7029

library(metafor)

rma(y=c(-0.00101, 0.023, 0.011), sei=c(0.03945, 0.016, 0.029), method="FE")
rma(y=c(-0.00101, 0.023, 0.011), sei=c(0.03945, 0.016, 0.029), method="DL")

DL(c(-0.00101, 0.023, 0.011), c(0.03945, 0.016, 0.029))

rs2433681       a       g       -0.0149 0.0225  0.5072  -+-     68.2    6.297   2       0.04292 -0.03706755     0.04442108      0.9704

CHR	POS	SNP	N	EFFECT_ALLELE	NON_EFFECT_ALLELE	EFFECT_ALLELE_FREQ	BETA	SE	r2	r2hat	P_VAL
2	169498916	rs2433681	1467	1	3	0.0978187	-0.1559	0.06133	0.004391	0.946	0.0118

CHR POS SNP STRAND EFFECT_ALLELE NON_EFFECT_ALLELE FREQ_EFFECT N BETA SE r2 PVALUE r2hat
2 169498916 rs2433681 + G A 0.926 1233 -0.014 0.029 0.00018 0.6296 0.9918

SNP	CHR	POS	Rsq	AL1	AL2	FREQ1	TRAIT	EFFECT	SE	H2	LOD	PVALUE
rs2433681	2	169498916	0.967	G	A	0.894	glucose_ND	0.009	0.044	0.002	0.008	0.8466

rma(y=c(-0.1559, 0.014, -0.009), sei=c(0.06133, 0.029, 0.044), method="FE")
rma(y=c(-0.1559, 0.014, -0.009), sei=c(0.06133, 0.029, 0.044), method="DL")


  # About ============================
  # @author: Adail Carvalho
  # @since : 2018-03-24
  # __________________________________

  #Required Libs ====
  checkAndInstallRequiredLib <- function(libname, extralib = NULL) {
    if (!require(libname)) {
      install.packages(libname)
    }

    if (!missing(extralib)) {
      if (!require(extralib)) {
        install.packages(extralib)
      }
    }
  }
  # Plotly: library that provides beaultiful chart tools.
  library(plotly)

  # Webshot: provides chart exports capabilities.
  library(webshot)

  # Matrices / DFrames headers ====
  modelHeader <- c("Coefficients", "Predictions", "RMSErrorRate", "Dataset", "YDependent", "XIndependent")
  lmHeader <- c("Observed Y","Predicted Y", "Residual E")

  #Error messages ====
  insuficientArgsException <- simpleError("Missing one or more required variables to proceed = ")
  insuficientArgsWarning <- "Missing one or more required variables to do the requested operation = "
  illegalArgsException <- simpleError("Invalid args: ")
  notSupportedException <- simpleError("Operation with the passed args are not supported yet.")

  # Main ====
  ' Entry point for our Linear Model program
   # input
   @filePath CSV file that represents the dataset
   @delimiter The CSV file delimiter (if its a TSV, then use "\t")
   @dependentVar Represents the Y column name in the dataset, where Y represents the dependent variable in the Linear Model.
        Colum Y must be numeric.
   @... Represents one or more datasets X columns names, where X represents the independent variables in the Linear Model.
        Columns X must be numerics.

   # returns
   @ model A List containing both Predictions and the Linear Model coefficients.
                 $Coefficients : list containing the values founded for the linear model coeficients
                 $Predictions  : matrix that shows the predicted values, the real values for Y and the E residuals
                 $RMSErrorRate : rating used to know how good our model is. A good model is always between -1< R <=1.
                 $Dataset: a matrix representation of the original dataset.
   # usage

   beearModel <- linearModel("C:/opt/beer_delivery.csv","\t", "Tempo", "NumCaixas", "Distancia")
   houses <- linearModel("C:/opt/houses_set.csv", ",", "SalePrice", "OverallQual", "YearBuilt", "MSSubClass", "BedroomAbvGr")

   # checking values

   houses$Coefficients
   houses$Predictions
  '
  linearModel <- function (filePath, delimiter, dependentVar, ...) {
    independentVarList <- list(...)

    if (is.null(dependentVar) || length(independentVarList) == 0) {
      stop(paste(insuficientArgsException, "{dependentVar, independentVarList}"))
    }

    matrixDataSet <- getFileToMatrix(filePath, delimiter)
    if (length(independentVarList) == 1) {
      model <- simpleLinearModel(matrixDataSet, dependentVar, independentVarList[[1]])
    } else {
      model <- multiLinearModel(matrixDataSet, dependentVar, independentVarList)
    }

    returnValue(model)
  }

  # Handle general operations to get the results
  # @returns results A list containing the coefficient values, RMSE error and the made predictions.
  simpleLinearModel <- function (matrixDataSet, dependentVar, independentVar) {
    if (!is.matrix(matrixDataSet)) {
      stop(illegalArgsException)
    }

    dependentIndex <- grep(dependentVar, colnames(matrixDataSet))
    independentIndex <- grep(independentVar, colnames(matrixDataSet))
    dependentVector <- matrixDataSet[,dependentIndex]
    independentVector <- matrixDataSet[,independentIndex]

    slope <- getSLRSlope(independentVector, dependentVector)
    intercept <- getSLRIntercept(independentVector, dependentVector, slope)

    resultModel <- doSimplePredictions(intercept, slope, independentVector, dependentVector)

    coefficientsMatrix <- matrix(nrow = 1, ncol = 2, dimnames = list(c(1),c("Intercept", independentVar)))
    coefficientsMatrix[,1] <- intercept
    coefficientsMatrix[,2] <- slope

    modelErrorRate <- estimateModelError(resultModel[,2])

    linearModel <- cbind(dependentVector,resultModel)
    colnames(linearModel) <- lmHeader

    # Gather generetade values
    model <- list(coefficientsMatrix, linearModel, modelErrorRate, matrixDataSet, dependentVar, independentVar)

    returnValue(model)
  }

  '
  Handle general operations to get the results
  @returns results A list containing the coefficient values, RMSE error and the made predictions.

  # Usage (directly)

  myModel <- multiLinearModel(dataMatrix, "Y var", as.list(c("var x1", "var x2", "var xn")))
  '
  multiLinearModel <- function (matrixDataSet, dependentVar, independentVarList) {
    if (!is.matrix(matrixDataSet)) {
      stop(illegalArgsException)
    }

    dependentColIndex <- grep(dependentVar, colnames(matrixDataSet))
    dependentMatrix <- matrixDataSet[,dependentColIndex]

    independentMatrix <- getIndependentVarsAsMatrix(matrixDataSet, independentVarList)

    levelsMatrix <- getRegressionLevelsMatrix(as.matrix(independentMatrix))

    xProductMatrix <- matrixTranposeAndMultiply(levelsMatrix, NULL)
    xProductMatrix <- getInverseMatrix(xProductMatrix)
    yProductMatrix <- matrixTranposeAndMultiply(levelsMatrix, as.matrix(dependentMatrix))

    coefficientsMatrix <- getMLRCoefficients(xProductMatrix, yProductMatrix, independentMatrix, independentVarList)

    resultModel <- doMultivariantPredictions(coefficientsMatrix, independentMatrix, dependentMatrix)

    modelErrorRate <- estimateModelError(resultModel[,2])

    linearModel <- cbind(dependentMatrix,resultModel)
    colnames(linearModel) <- lmHeader

    model <- list(coefficientsMatrix, linearModel, modelErrorRate, matrixDataSet, dependentVar, independentVarList)
    model <- handleDataHeaders(model, modelHeader)

    returnValue(model)
  }

  # Linear Model ====

  # Estimates the values for Y, using the calculated coefficients.
  # Apllies the following rule: ~Yi : B0 + (B1 * Xi)
  # Where ~Yi = estimated value for dependent value Y, B0 = Slope coefficient, B1 = Intercept coefficient, Xi = observed value for X,
  # and i, the iterator value over the X independent vector
  # returns a Matrix with the estimated values for Y and the residuals E, where E = real value of Y - estimated value for Y
  doSimplePredictions <- function(coefficientB0, coefficientB1, independentVector, dependentVector) {
    predictedY <- vector("numeric", length(dependentVector))
    residuals <- vector("numeric", length(independentVector))

    for (i in 1:length(independentVector)) {
      predictedY[i] <- coefficientB0 + (coefficientB1 * independentVector[i])
    }

    residualVector <- dependentVector - predictedY
    returnValue(cbind(predictedY, residualVector))
  }

  # Estimates the values for Y, using the calculated coefficients.
  # Apllies the following rule: ~Y : B0 + B1(X1 - ~X1) + B2(X2 - ~X2) + ... + Bn(Xn - ~Xn)
  # Where ~Y = estimated value for dependent value Y, Bi = Coeficient values, Xi = observed value for X, ~Xi = mean of X variable.
  # returns a Matrix with the estimated values for Y and the residuals E, where E = real value of Y - estimated value for Y
  doMultivariantPredictions <- function(coefficientsMatrix, independentMatrix, dependentMatrix = NULL) {
    predictedY <- matrix(0, nrow = nrow(independentMatrix), ncol = 1)
    residualMatrix <- matrix(0 , nrow = nrow(independentMatrix), ncol = 1)

    for (r in 1:nrow(independentMatrix)) {
      for (coe in 2:ncol(coefficientsMatrix)) {
        predictedY[r,1] <- predictedY[r,1] + (coefficientsMatrix[,coe] * independentMatrix[r,coe - 1])
      }
      predictedY[r,1] <- predictedY[r,1] + coefficientsMatrix[,1]
    }

    predictions <-NULL
    if (missing(dependentMatrix)) {
      predictions <- predictedY
    } else {
      residualMatrix <- dependentMatrix - predictedY
      predictions <- cbind(predictedY, residualMatrix)
    }

    returnValue(predictions)
  }

  # Matrices general operations ====
  getMLRCoefficients <- function(xProductMatrix, yProductMatrix, independentMatrix, xLabels) {
    avgIndMat <- colMeans(independentMatrix)

    coefficientsMatrix <- matrixMultiply(xProductMatrix, yProductMatrix)
    coefficientsMatrix <- t(coefficientsMatrix)
    slope <- coefficientsMatrix[,1]

    for(c in 2:ncol(coefficientsMatrix)) {
      slope <- slope + (coefficientsMatrix[,c] * (-avgIndMat[c - 1]))
    }
    coefficientsMatrix[,1] <- slope
    colnames(coefficientsMatrix) <-c("[Intercept]", xLabels)
    returnValue(coefficientsMatrix)

  }
  # Returns the (matriz)-¹ of a given matrix.
  # inv(x) is a funtion provided by the package matlib*
  getInverseMatrix <- function(matriz) {
    invertedMatrix <- solve(matriz)
    returnValue(invertedMatrix)
  }

  # Returns the mean of independent variables in multivariant linear model
  getRegressionLevelsMatrix <- function(matriz) {
    avgIndependentsVarMatrix <- t(as.matrix(colMeans(matriz)))
    levelsMatrix <- cbind(getX0Dimension(matriz), matriz)

    for (c in colnames(matriz)) {
      x <- matriz[,c]
      l <- vector(class(x), length(x))
      for (i in 1:length(x)) {
        l[i] <- x[i] - avgIndependentsVarMatrix[,c]
      }
      levelsMatrix[,c] <- l
    }
    returnValue(levelsMatrix)
  }

  # Represents the x0 coeficient in the lm equation, where x0 = 1. In the matrix of independent values, it is placed in the first column.
  getX0Dimension <- function(matriz) {
    numRows <- nrow(matriz)
    returnValue(matrix(rep(1, numRows), dimnames = list(c(1:numRows), c("X0"))))
  }

  # Return product of two matrices
  matrixMultiply <- function(matrixA, matrixB) {
    resultMatrix <- matrix(0, nrow = nrow(matrixA), ncol = ncol(matrixB))
    for (r in 1:nrow(matrixA)) {
      for (c in 1:ncol(matrixB)) {
        for (kol in 1:ncol(matrixA)) {
          resultMatrix[r,c] <- resultMatrix[r,c] + (matrixA[r,kol] * matrixB[kol, c])
        }
      }
    }

    returnValue(resultMatrix)

  }

  # Returns the product of matrices
  matrixTranposeAndMultiply <- function(matriz, matrixB) {
    transpMatrix <- t(matriz)

    if (is.null(matrixB)) {
      matrixY <- matriz
    } else {
      matrixY <- matrixB
    }

    returnValue(matrixMultiply(transpMatrix, matrixY))
  }

  # Linear Regression Rules Application ====
  # Estimate erros using RootMeanSquareError(RMSE).
  # @residualVector Vector that holds the E residuals of our model.
  estimateModelError <- function(residualVector) {
    rmse <- sqrt(sum(residualVector ** 2) / length(residualVector))
    returnValue(rmse)
  }

  # Returns SLR Slope value
  getSLRSlope <- function(independentVector, dependentVector) {
    levelsVectorX <- independentVector - mean(independentVector)
    levelsVectorY <- dependentVector - mean(dependentVector)

    numeratorB1 <- sum(levelsVectorX * levelsVectorY)
    denominatorB1 <- sum(levelsVectorX ** 2)
    slope <- numeratorB1 / denominatorB1

    returnValue(slope)
  }

  # Returns SLR Intercept value
  getSLRIntercept <- function (independentVector, dependentVector, slope) {
    intercept <- mean(dependentVector) - (slope * mean(independentVector))
    returnValue(intercept)
  }

  # Utils ====
  getIndependentVarsAsMatrix <- function(matrixData, independentVarList) {
    independentMatrix <- matrix(0, nrow = nrow(matrixData), ncol = length(independentVarList))
    colnames(independentMatrix) <- independentVarList

    for (i in independentVarList) {
      name <- grep(i, colnames(matrixData))
      independentMatrix[,i] <- matrixData[,name]
    }
    returnValue(independentMatrix)
  }

  getFileToMatrix <- function(filePath, delimiter) {
    rawDataSet <-  read.csv(filePath, sep = delimiter, header = TRUE)
    returnValue(data.matrix(rawDataSet))
  }

  ' Export charts as images.
   @p A chart object
   @filename Name of the file to be saved.
   @ext File extension
  '
  exportChartAsImg <- function(p, filename, ext) {
    export(p, paste(filename, ext))
  }

  handleDataHeaders <- function(data, header) {
    names(data) <- modelHeader
    returnValue(data)
  }

  # Charts ####
  # Add to a chart labels and title
  applyLayout <- function(p, xLabel, yLabel, chartTitle) {
    title <- NULL
    if (missing(chartTitle)) {
      title <- paste(yLabel, " by ", xLabel)
    } else {
      title <- chartTitle
    }

    p <- layout(p, title = title,
                xaxis = list(title = xLabel),
                yaxis = list (title = yLabel))
    returnValue(p)

  }

  'Export models as simple X vs Y line charts, from a given model.
  @ model A LinearRegressor object.
  @ dirToSave The directory where the plots will be exported
  '
  createChartsFromModel <- function(model, dirToSave = NULL) {
    for (x in model$XIndependent) {
      if (isTRUE(grepl('^[0-9]', x))) {
        x <- paste("X", x, sep = "")
      }

      print(paste("Generating chart for => ", x))

      p <- lmYXLineChart(model$Dataset[,x],
                         model$Dataset[,model$YDependent],
                         model$Predictions[,"Predicted Y"],
                         model$Predictions[,"Residual E"],
                         x,
                         model$YDependent)

      exportChartAsImg(p, paste(dirToSave, model$YDependent, " by ", x), ".png")
    }
  }

  # Creates a line chart, by ploting y in the y axis and x in the x axis.
  # lmYXLineChart(TipoMoradores, PrecoVenda, PrecoPrevisto, ResiduoModelo, "TipoMoradores", "PrecoVenda")
  lmYXLineChart <- function(xData, yData, yPredicted, eResid, xLabel, yLabel) {
    xName <- xLabel
    yName <- yLabel
    if (missing(xName)) {
      xName <- "X"
    }

    if (missing(yName)) {
      yName <- "Y"
    }

    chartFrame <- data.frame(xData, yData, yPredicted, eResid)
    aggData <- aggregate(chartFrame, by = list(chartFrame$xData), FUN = mean)

    lmPlot<- plot_ly(aggData, x = ~aggData$xData, name = xName)

    lmPlot <- add_trace(lmPlot, y= ~aggData$yData, name = yName, type = 'scatter', mode = 'lines')
    lmPlot <- add_trace(lmPlot, y = ~aggData$yPredicted, name = paste(yName, "(Pred)"), type = 'scatter', mode = 'lines')
    lmPlot <- add_trace(lmPlot, y = ~aggData$eResid, name = 'Residual', type = 'scatter', mode = 'markers')

    lmPlot <- applyLayout(lmPlot, xName, yName)
    returnValue(lmPlot)

  }


# have all phylogenies in one file
# have a file with the chromosome and window for each tree in the correct order
# have a popmap with the individual names and groupings wished to test
# have the outgroup labeled once in the popmap as "outgroup"

library(ape)
library(phytools)

options(scipen=999)

# read in trees, info, and popmap for 50kbp trees
x_trees <- read.tree("certhia_50kbp.trees")
x_info <- read.table("certhia_50kbp_tree_info.txt", sep="\t", stringsAsFactors=F)
x_popmap <- read.table("gsi_popmap.txt", sep="\t", stringsAsFactors=F)
x_output <- "b_gsi_50kbp_output.txt"

# define outgroup
outgroup <- x_popmap[x_popmap[,2] == "outgroup", 1]

# remove outgroup from popmap
x_popmap <- x_popmap[x_popmap[,2] != "outgroup", ]

# write initial line of output
write(c("chr", "start", "end", "pop", "gsi"), file=x_output, sep="\t", ncolumns=5)

# loop for each tree
for(a in 1:length(x_trees)) {
	# select the tree
	x <- x_trees[a][[1]]
	# reroot
	x <- midpoint.root(x)
	x <- root(x, outgroup, resolve.root=T)

	# define the groups
	groups <- unique(x_popmap[,2])

	# loop for each group of interest
	for(g in 1:length(groups)) {
		# define the group of interest
		group_of_interest <- x_popmap[x_popmap[,2] == groups[g],1]

		# calculate MRCA of group of interest
		group_mrca <- getMRCA(x, tip = group_of_interest)

		# calculate number of ndoes to reach common ancestor for each individual in group
		# and then remove redundant nodes
		nodes_needed <- c()
		for(b in 1:length(group_of_interest)) {
			# what is the number of this individual?
			b_number <- match(group_of_interest[b], x$tip.label)

			# loop throup edge table until reaching MRCA
			while_loop <- 0
			while(while_loop != group_mrca) {
				# add node
				nodes_needed <- c(nodes_needed, x$edge[x$edge[,2]==b_number,1])

				# change new number to that node and update the while_loop object to the node
				b_number <- x$edge[x$edge[,2]==b_number,1]
				while_loop <- b_number
			}
			# add the MRCA to the nodes_needed object
			nodes_needed <- c(nodes_needed, group_mrca)

			# only keep unique nodes
			nodes_needed <- unique(nodes_needed)
		}

		# calculate gsi

		# gs
		gs <- (length(group_of_interest) - 1) / length(nodes_needed)

		# max gs = 1
		max_gs <- 1

		# min gs = minimum number of nodes to connect all individuals (n - 1) / total number of nodes
		min_gs <- (length(group_of_interest) - 1) / length(unique(x$edge[,1]))

		# equation 4 of Cummings et al. 2008 (GSI paper)
		gsi <- (gs  - min_gs) / (max_gs - min_gs)

		# write output
		write(c(x_info[a,1], x_info[a,2], x_info[a,3], groups[g], gsi), file=x_output, sep="\t", ncolumns=5, append=T)
	}
}

# read in trees, info, and popmap for 100kbp trees
x_trees <- read.tree("certhia_100kbp.trees")
x_info <- read.table("certhia_100kbp_tree_info.txt", sep="\t", stringsAsFactors=F)
x_popmap <- read.table("gsi_popmap.txt", sep="\t", stringsAsFactors=F)
x_output <- "b_gsi_100kbp_output.txt"

# define outgroup
outgroup <- x_popmap[x_popmap[,2] == "outgroup", 1]

# remove outgroup from popmap
x_popmap <- x_popmap[x_popmap[,2] != "outgroup", ]

# write initial line of output
write(c("chr", "start", "end", "pop", "gsi"), file=x_output, sep="\t", ncolumns=5)

# loop for each tree
for(a in 1:length(x_trees)) {
	# select the tree
	x <- x_trees[a][[1]]
	# reroot
	x <- midpoint.root(x)
	x <- root(x, outgroup, resolve.root=T)

	# define the groups
	groups <- unique(x_popmap[,2])

	# loop for each group of interest
	for(g in 1:length(groups)) {
		# define the group of interest
		group_of_interest <- x_popmap[x_popmap[,2] == groups[g],1]

		# calculate MRCA of group of interest
		group_mrca <- getMRCA(x, tip = group_of_interest)

		# calculate number of ndoes to reach common ancestor for each individual in group
		# and then remove redundant nodes
		nodes_needed <- c()
		for(b in 1:length(group_of_interest)) {
			# what is the number of this individual?
			b_number <- match(group_of_interest[b], x$tip.label)

			# loop throup edge table until reaching MRCA
			while_loop <- 0
			while(while_loop != group_mrca) {
				# add node
				nodes_needed <- c(nodes_needed, x$edge[x$edge[,2]==b_number,1])

				# change new number to that node and update the while_loop object to the node
				b_number <- x$edge[x$edge[,2]==b_number,1]
				while_loop <- b_number
			}
			# add the MRCA to the nodes_needed object
			nodes_needed <- c(nodes_needed, group_mrca)

			# only keep unique nodes
			nodes_needed <- unique(nodes_needed)
		}

		# calculate gsi

		# gs
		gs <- (length(group_of_interest) - 1) / length(nodes_needed)

		# max gs = 1
		max_gs <- 1

		# min gs = minimum number of nodes to connect all individuals (n - 1) / total number of nodes
		min_gs <- (length(group_of_interest) - 1) / length(unique(x$edge[,1]))

		# equation 4 of Cummings et al. 2008 (GSI paper)
		gsi <- (gs  - min_gs) / (max_gs - min_gs)

		# write output
		write(c(x_info[a,1], x_info[a,2], x_info[a,3], groups[g], gsi), file=x_output, sep="\t", ncolumns=5, append=T)
	}
}