#' Simulate data to test tryx
#'
#' # Create data using model here: https://www.draw.io/#G1VqjTd4iH2de7sXI4PhUrxazBnYLduH_w. Create summary data for all necessary traits, then perform tryx scan
#' x = exposure
#' y = outcome
#' u1 = confounder of x and y
#' u2 = mediator of horizontal pleiotropy from gx to y. One for each gx
#' u3 = mediator between x and y
#'
#' @param nid = 10000 Number of samples
#' @param ngx = 30 Number of direct instruments to x
#' @param ngu1 = 30 Number of instruments influencing confounder of x and y
#' @param ngu2 = 30 Number of instruments per mediating
#' @param nu2 = 2 Number of gx that are pleiotropic
#' @param ngu3 = 30 Number of instruments for u3 mediator
#' @param vgx = 0.2 Variance explained by gx instruments
#' @param vgu1 = 0.6 Variance explained by u1 instruments
#' @param vgu2 = 0.2 Variance explained by u2 instruments
#' @param vgu3 = 0.2 Variance explained by u3 instruments
#' @param bxy = 0 Causal effect of x on y
#' @param bu1x = 0.6 Effect of u1 on x
#' @param bu1y = 0.4 Effect of u1 on y
#' @param bxu3 = 0.3 Effect of x on u3
#' @param bu3y = 0 Effect of u3 on y
#' @param vgxu2 = 0.2 Variance explained by each gx instrument on each u2 mediator
#' @param vu2y = 0.2 Variance explained by all u2 mediators on y
#' @param ngxu3 = 0 Number of gx instruments that pleiotropically associate with u3 mediator
#' @param vgxu3y = 0 Variance explained by all gx variants directly on u3 mediator
#' @param mininum_instruments = 10 Minimum number of instruments required to have been detected to run simulation
#' @param instrument_threshold = "bonferroni" Threshold, either numeric or 'bonferroni'
#' @param outlier_threshold = "bonferroni" Threshold, either numeric or 'bonferroni'
#' @param outliers_known = "detected" Either detected = using radial mr to find heterogeneity outliers; known = adjust for all known invalid instruments; all = adjust for all instruments
#' @param directional_bias = FALSE Is the pleiotropic effect randomly centered around 0 (FALSE) or does it have a non-0 mean (TRUE)
#'
#' @export
#' @return list for tryx.analyse
tryx.simulate <- function(nid = 10000, ngx = 30, ngu1 = 30, ngu2 = 30, nu2 = 2, ngu3 = 30, vgx = 0.2, vgu1 = 0.6, vgu2 = 0.2, vgu3 = 0.2, bxy = 0, bu1x = 0.6, bu1y = 0.4, bxu3 = 0.3, bu3y = 0, vgxu2 = 0.2, vu2y = 0.2, ngxu3 = 0, vgxu3=0, mininum_instruments = 10, instrument_threshold = "bonferroni", outlier_threshold = "bonferroni", outliers_known = "detected", directional_bias = FALSE)
{
	out <- list()

	message("Generating genetic effects")

	gx <- make_geno(nid, ngx, 0.5)
	gu1 <- make_geno(nid, ngu1, 0.5)
	gu2 <- make_geno(nid, ngu2 * nu2, 0.5)
	gu3 <- make_geno(nid, ngu3, 0.5)
	u1 <- make_phen(choose_effects(ngu1, vgu1), gu1)
	x <- make_phen(c(abs(choose_effects(ngx, vgx)), bu1x), cbind(gx, u1))
	if(ngxu3 > 0)
	{
		u3 <- make_phen(c(choose_effects(ngu3, vgu3), bxu3, choose_effects(ngxu3, vgxu3)), cbind(gu3, x, gx[,sample(1:ngx, ngxu3)]))
	} else {
		u3 <- make_phen(c(choose_effects(ngu3, vgu3), bxu3), cbind(gu3, x))
	}

	# Some number of the direct gx SNPs also influence the outcome via a mediator
	# This is mediated pleiotropy

	message("Creating potential pleiotropy paths")

	u2 <- matrix(rnorm(nid * nu2), nid, nu2)
	for(i in 1:nu2)
	{
		message("SNP ", i, " is pleiotropic")
		u2[,i] <- make_phen(
			c(choose_effects(ngu2, vgu2), abs(choose_effects(1, vgxu2))),
			cbind(gu2[,((i-1) * ngu2 + 1):(ngu2 * i)], gx[,i])
		)
	}

	message("Creating outcome")

	bu2y <- choose_effects(nu2, vu2y)

	if(directional_bias)
	{
		message("U2 bias is directional")
		bu2y <- abs(bu2y)
	}
	y <- make_phen(
		c(bxy, bu3y, bu1y, bu2y),
		cbind(x, u3, u1, u2)
	)

	# Single genotype data source for simplicity

	G <- cbind(gx, gu1, gu2, gu3)
	G_annotation <- tibble(
		snp=1:ncol(G),
		origin1=c(
			rep("x", ngx),
			rep("u1", ngu1),
			rep(paste0("u2_",1:nu2), each=ngu2),
			rep("u3", ngu3)
		),
		origin2=c(
			rep("x", ngx),
			rep("u1", ngu1),
			rep("u2", each=ngu2*nu2),
			rep("u3", ngu3)
		)
	)

	# How do we select instruments?

	if(instrument_threshold == "bonferroni")
	{
		instrument_threshold <- 0.05 / ncol(G)
	} else {
		stopifnot(is.numeric(instrument_threshold))
	}
	message("instrument_threshold = ", instrument_threshold)

	# Get summary set

	message("Getting instruments")

	out$dat_all <- get_effs(x, y, G, "X", "Y")
	out$dat <- subset(out$dat_all, pval.exposure < instrument_threshold)
	stopifnot(nrow(out$dat) > mininum_instruments)
	out$dat$mr_keep <- TRUE
	G_annotation$detected <- G_annotation$snp %in% out$dat$SNP
	message("Detected ", nrow(out$dat), " associations for x")

	## Determine valid instruments

	if(vu2y == 0)
	{
		valid <- 1:ngx
	} else {
		valid <- (1:ngx)[! (1:ngx %in% 1:nu2)]
	}

	if(bu1y == 0)
	{
		valid <- c(valid, (ngx+1):(ngx+ngu1))
	}
	message("Valid instruments: ", paste(valid, collapse=","))
	G_annotation$valid <- G_annotation$snp %in% valid

	## known invalid instruments
	invalid <- which(! 1:ncol(G) %in% valid)

	out$valid <- valid
	out$invalid <- invalid

	# Get outliers

	no_outlier_flag <- FALSE
	if(outliers_known == "known")
	{
		message("Assuming knowledge of all outliers")
		outliers <- out$dat$SNP[out$dat$SNP %in% invalid]
	} else if(outliers_known %in% c("detected", "all")) {
		message("Detecting outliers")
		radial <- RadialMR::ivw_radial(RadialMR::format_radial(out$dat$beta.exposure, out$dat$beta.outcome, out$dat$se.exposure, out$dat$se.outcome, out$dat$SNP), ifelse(outlier_threshold == "bonferroni", 0.05/nrow(out$dat), outlier_threshold), weights=3)
		if(radial$outliers[1] == "No significant outliers")
		{
			message("No significant outliers detected")
			outliers <- as.character(out$dat$SNP[out$dat$SNP %in% invalid])
			if(length(outliers) == 0) outliers <- 1
			no_outlier_flag <- TRUE
		} else {
			outliers <- as.character(sort(radial$outliers$SNP))
			message("Number of outliers: ", length(outliers))
		}
	} else {
		stop("outliers_known must be 'known', 'detected' or 'all'")
	}

	nout <- length(outliers)
	outliers <- subset(out$dat, SNP %in% outliers)$SNP
	out$outliers <- outliers
	if(outliers_known == "all")
	{
		message("Treating all variants as outliers")
		outliers <- out$dat$SNP
	}
	# out$id_list <- c(paste0("U1_", 1:nu1), paste0("U2_", 1:nu2))
	message("Outliers to be analysed: ", paste(outliers, collapse=","))

	message("Perform outlier scan")

	U <- cbind(u1, u2, u3)
	colnames(U) <- c("U1", paste0("U2_", 1:ncol(u2)), "U3")
	outlier_scan <- list()
	for(i in 1:ncol(U))
	{
		outlier_scan[[colnames(U)[i]]] <- gwas(U[,i], G[,outliers, drop=FALSE])
		outlier_scan[[colnames(U)[i]]]$SNP <- outliers
		outlier_scan[[colnames(U)[i]]]$outcome <- colnames(U)[i]
	}
	outlier_scan <- bind_rows(outlier_scan)
	out$search <- tibble::tibble(
		SNP=outlier_scan$SNP,
		beta.outcome=outlier_scan$bhat,
		se.outcome=outlier_scan$se,
		pval.outcome=outlier_scan$pval,
		id.outcome=outlier_scan$outcome,
		outcome=outlier_scan$outcome,
		mr_keep=TRUE,
		effect_allele.outcome="A",
		other_allele.outcome="G",
		eaf.outcome=0.5
	)
	out$search$sig <- out$search$pval.outcome < 5e-8
	message("Found the following candidate traits: ", paste(unique(subset(out$search, sig)$id.outcome), collapse=","))

	message("Organise data for mvmr")

	traitlist <- as.character(unique(subset(out$search, sig)$id.outcome))
	PHEN <- cbind(x, U[,traitlist])
	PHEN <- lapply(seq_len(ncol(PHEN)), function(i) PHEN[,i])
	names(PHEN) <- c("x", traitlist)
	message("Analysing ", length(PHEN), " traits: ", paste(names(PHEN), collapse=","))

	snplist <- sapply(1:length(PHEN), function(x) {
		subset(gwas(PHEN[[x]], G), pval < instrument_threshold)$snp
	}) %>% unlist() %>% sort %>% unique
	message("Found ", length(snplist), " unique instruments")

	message("Perform mvmr")
	out$mvres <- try({
		mvdat <- make_mvdat(PHEN, y, G[,snplist])
		mvres <- mv_multiple(mvdat)
		mvres$result$exposure <- c("x", traitlist)
		mvres
	})

	message("Building effects for ", length(traitlist), " trait(s)")

	## get effects
	ux <- list()
	uy <- list()
	for(i in traitlist)
	{
		message(i)
		ux[[i]] <- get_effs(U[,i], x, G, i, "X") %>% subset(pval.exposure < instrument_threshold)
		uy[[i]] <- get_effs(U[,i], y, G, i, "Y") %>% subset(pval.exposure < instrument_threshold)
		if(nrow(ux[[i]]) > 0)
		{
			ux[[i]]$effect_allele.exposure <- "A"
			ux[[i]]$other_allele.exposure <- "G"
			ux[[i]]$effect_allele.outcome <- "A"
			ux[[i]]$other_allele.outcome <- "G"
			ux[[i]]$eaf.exposure <- 0.5
			ux[[i]]$eaf.outcome <- 0.5
		}

		if(nrow(uy[[i]]) > 0)
		{
			uy[[i]]$effect_allele.exposure <- "A"
			uy[[i]]$other_allele.exposure <- "G"
			uy[[i]]$effect_allele.outcome <- "A"
			uy[[i]]$other_allele.outcome <- "G"
			uy[[i]]$eaf.exposure <- 0.5
			uy[[i]]$eaf.outcome <- 0.5
		}
	}

	out$candidate_instruments <- subset(bind_rows(ux), select=c(
		SNP, beta.exposure, se.exposure, id.exposure, exposure, effect_allele.exposure, other_allele.exposure, eaf.exposure, pval.exposure
	))

	message("Removing known outliers from candidate instruments")

	out2 <- subset(out$search, sig)
	out$candidate_instruments <- group_by(out$candidate_instruments, id.exposure) %>%
		do({
			x <- .
			y <- subset(out2, id.outcome == x$id.exposure[1])
			x <- subset(x, !SNP %in% y$SNP)
			x
		})

	message("MR of candidates on outcome")

	out$candidate_outcome <- subset(bind_rows(uy), select=c(
		SNP, beta.outcome, se.outcome, id.outcome, outcome, effect_allele.outcome, other_allele.outcome, eaf.outcome, pval.outcome
	))

	# out$candidate_outcome_dat <- suppressMessages(harmonise_data(out$candidate_instruments, out$candidate_outcome))
	out$candidate_outcome_dat <- harmonise_data(out$candidate_instruments, out$candidate_outcome)
	out$candidate_outcome_mr <- suppressMessages(mr(out$candidate_outcome_dat, method_list="mr_ivw"))

	message("MR of candidates on exposure")

	out$candidate_exposure <- subset(bind_rows(ux), select=c(
		SNP, beta.outcome, se.outcome, id.outcome, outcome, effect_allele.outcome, other_allele.outcome, eaf.outcome, pval.outcome
	))

	# out$candidate_exposure_dat <- suppressMessages(harmonise_data(out$candidate_instruments, out$candidate_exposure))
	out$candidate_exposure_dat <- harmonise_data(out$candidate_instruments, out$candidate_exposure)
	out$candidate_exposure_mr <- suppressMessages(mr(out$candidate_exposure_dat, method_list="mr_ivw"))

	message("Finalising")
	out$G_annotation <- G_annotation
	out$simulation <- list()
	out$simulation$no_outlier_flag <- no_outlier_flag
	out$simulation$nid <- nid
	out$simulation$ngx <- ngx
	out$simulation$ngu1 <- ngu1
	out$simulation$ngu2 <- ngu2
	out$simulation$nu2 <- nu2
	out$simulation$ngu3 <- ngu3
	out$simulation$vgx <- vgx
	out$simulation$vgu1 <- vgu1
	out$simulation$vgu2 <- vgu2
	out$simulation$vgu3 <- vgu3
	out$simulation$bxy <- bxy
	out$simulation$bu1x <- bu1x
	out$simulation$bu1y <- bu1y
	out$simulation$bxu3 <- bxu3
	out$simulation$bu3y <- bu3y
	out$simulation$vgxu2 <- vgxu2
	out$simulation$vu2y <- vu2y
	out$simulation$ngxu3 <- ngxu3
	out$simulation$vgxu3 <- vgxu3
	out$simulation$mininum_instruments <- mininum_instruments
	out$simulation$instrument_threshold <- instrument_threshold
	out$simulation$outlier_threshold <- outlier_threshold
	out$simulation$outliers_known <- outliers_known
	out$simulation$directional_bias = directional_bias
	out$simulation$n_instruments <- sum(G_annotation$detected)
	out$simulation$n_valid_instruments <- sum(G_annotation$detected * G_annotation$valid)

return(out)
}

# #' Simulate data to test tryx
# #'
# #' Choose a sample size, number of horizontal pleiotropy paths, and number of confounders. Create summary data for all necessary traits, then perform tryx scan
# #'
# #' @param nid Sample size to simulate
# #' @param nu1 Number of traits which influence both X and Y
# #' @param nu2 Number of traits which influence Y
# #' @param bxu3 Effect of x on mediator
# #' @param bu3y Effect of mediator on y
# #' @param bxy Causal effect of X on Y
# #' @param outliers_known Assume that all outliers are detected (default = 'known'). But can also be 'detected', where we use heterogeneity based outlier detection, or 'all' where we try to adjust for every outlier regardless of heterogeneity
# #' @param directional_bias Is the outlier typically in one direction (i.e. unbalanced pleiotropy?)
# #' @param outlier_threshold Either 'nominal' or 'bonferroni'
# #' @param single_pleiotropy_trait = FALSE
# #'
# #' @export
# #' @return Results from simulations and tryx scan
# tryx.simulate <- function(nid, nu1, nu2, bxu3=0, bu3y=0, bxy=3, outliers_known=c("known", "detected", "all")[1], directional_bias=FALSE, outlier_threshold = 'nominal', single_pleiotropy_trait = FALSE, debug=FALSE, ngu4=10, ngx=30, bu4x=0, bu4y=0)
# {
# 	# scenario 1 - confounder g -> u; u -> x; u -> y
# 	# scenario 2 - pleiotropy g -> u -> y
# 	# scenario 3 - mediator
# 	# scenario 4 -

# 	out <- list()

# 	cpg <- require(simulateGP)
# 	if(!cpg)
# 	{
# 		stop("Please install the simulateGP package\ndevtools::install_github('explodecomputer/simulateGP')")
# 	}

# 	ngu1 <- 30
# 	ngu2 <- 30
# 	ngu3 <- 30
# 	nu3 <- 1
# 	nu4 <- 1
# 	nu <- nu1 + nu2 + nu3
# 	stopifnot(ngx >= nu)

# 	gx <- make_geno(nid, ngx, 0.5)
# 	out$true_outliers <- 1:nu

# 	# Effect sizes
# 	bgu1 <- lapply(1:nu1, function(x) runif(ngu1))
# 	bgu2 <- lapply(1:nu2, function(x) runif(ngu2))
# 	bgu3 <- runif(ngu3)
# 	bgx <- runif(ngx)

# 	bu1x <- out$bu1x <- rnorm(nu1)
# 	bu1y <- out$bu1y <- rnorm(nu1)
# 	bu2y <- out$bu2y <- rnorm(nu2)
# 	if(directional_bias)
# 	{
# 		bu1x <- out$bu1x <- abs(bu1x)
# 		bu1y <- out$bu1y <- abs(bu1y)
# 		bu2y <- out$bu2y <- abs(bu2y)
# 	}
# 	out$bxu3 <- bxu3
# 	bu3y <- out$bu3y <- bu3y
# 	bxy <- out$bxy <- bxy

# 	# create outlier traits
# 	gu1 <- list()
# 	u1 <- matrix(rnorm(nid * ngx), nid, ngx)
# 	if(nu1 > 0)
# 	{
# 		for(i in 1:nu1)
# 		{
# 			gu1[[i]] <- cbind(gx[,i], make_geno(nid, ngu1-1, 0.5))
# 			if(single_pleiotropy_trait)
# 			{
# 				u1[,1] <- u1[,1] + gu1[[i]] %*% bgu1[[i]]
# 			} else {
# 				u1[,i] <- gu1[[i]] %*% bgu1[[i]] + rnorm(nid)
# 			}
# 		}
# 	}

# 	gu2 <- list()
# 	u2 <- matrix(rnorm(nid * ngx), nid, ngx)
# 	if(nu2 > 0)
# 	{
# 		for(j in 1:nu2)
# 		{
# 			gu2[[j]] <- cbind(gx[,i+j], make_geno(nid, ngu1-1, 0.5))
# 			if(single_pleiotropy_trait)
# 			{
# 				u2[,1] <- u2[,1] + gu2[[j]] %*% bgu2[[j]]
# 			} else {
# 				u2[,j] <- gu2[[j]] %*% bgu2[[j]] + rnorm(nid)
# 			}
# 		}
# 	}

# 	u4 <- rnorm(nid)
# 	if(ngu4 > 0)
# 	{
# 		u4 <- u4 + gx[,1:ngu4] %*% runif(ngu4)
# 	}
# 	x <- drop(gx[,c((ngu4+1):ngx)] %*% bgx[c((ngu4+1):ngx)] + u4 %*% bu4x + u1 %*% bu1x + rnorm(nid))

# 	gu3 <- list(make_geno(nid, ngu3, 0.5))
# 	u3 <- matrix(gu3[[1]] %*% bgu3 + x * bxu3 + rnorm(nid), nid, 1)

# 	y <- drop(x * bxy + u1 %*% bu1y + u2 %*% bu2y + u3 * bu3y + u4 %*% bu4y + rnorm(nid))

# 	out$dat <- get_effs(x, y, gx, "X", "Y")
# 	out$dat$mr_keep <- TRUE
# 	no_outlier_flag <- FALSE

# 	if(outliers_known == "known")
# 	{
# 		message("Assuming knowledge of all outliers")
# 		outliers <- as.character(c(1:nu))
# 	} else if(outliers_known %in% c("detected", "all")) {
# 		message("Detecting outliers")
# 		radial <- RadialMR::ivw_radial(RadialMR::format_radial(out$dat$beta.exposure, out$dat$beta.outcome, out$dat$se.exposure, out$dat$se.outcome, out$dat$SNP), ifelse(outlier_threshold == "nominal", 0.05, 0.05/nrow(out$dat)))
# 		if(radial$outliers[1] == "No significant outliers")
# 		{
# 			message("No significant outliers detected")
# 			outliers <- as.character(c(1:nu))
# 			no_outlier_flag <- TRUE
# 		} else {
# 			outliers <- as.character(sort(radial$outliers$SNP))
# 			message("Number of outliers: ", length(outliers))
# 		}
# 	} else {
# 		stop("outliers_known must be known, detected or all")
# 	}
# 	# output$radialmr <- radial
# 	nout <- length(outliers)
# 	outliers <- subset(out$dat, SNP %in% outliers)$SNP
# 	out$outliers <- outliers
# 	if(outliers_known == "all")
# 	{
# 		message("Treating all variants as outliers")
# 		outliers <- 1:ngx
# 	}
# 	out$id_list <- c(paste0("U1_", 1:nu1), paste0("U2_", 1:nu2))

# 	# temp1 <- gwas(u1, gx[,1:2])

# 	temp <- list()
# 	for(i in 1:nu1)
# 	{
# 		temp[[i]] <- gwas(u1[,i], gx[,outliers, drop=FALSE])
# 		temp[[i]]$SNP <- outliers
# 		temp[[i]]$outcome <- paste0("U1_", i)
# 	}
# 	for(i in 1:nu2)
# 	{
# 		temp[[i + nu1]] <- gwas(u2[,i], gx[,outliers, drop=FALSE])
# 		temp[[i + nu1]]$SNP <- outliers
# 		temp[[i + nu1]]$outcome <- paste0("U2_", i)
# 	}
# 	for(i in 1:nu3)
# 	{
# 		temp[[i + nu1 + nu2]] <- gwas(u3[,i], gx[,outliers, drop=FALSE])
# 		temp[[i + nu1 + nu2]]$SNP <- outliers
# 		temp[[i + nu1 + nu2]]$outcome <- paste0("U3_", i)
# 	}
# 	for(i in 1:nu4)
# 	{
# 		temp[[i + nu1 + nu2 + nu3]] <- gwas(u4[,i], gx[,outliers, drop=FALSE])
# 		temp[[i + nu1 + nu2 + nu3]]$SNP <- outliers
# 		temp[[i + nu1 + nu2 + nu3]]$outcome <- paste0("U4_", i)
# 	}

# 	temp <- bind_rows(temp)

# 	out$search <- tibble::data_frame(SNP=temp$SNP, beta.outcome=temp$bhat, se.outcome=temp$se, pval.outcome=temp$pval, id.outcome=temp$outcome, outcome=temp$outcome, mr_keep=TRUE, effect_allele.outcome="A", other_allele.outcome="G", eaf.outcome=0.5)
# 	out$search$sig <- out$search$pval.outcome < 5e-8

# 	## Multivariable MR
# 	# Do mvmr with just the outlier detected traits

# 	traitlist <- as.character(unique(subset(out$search, sig)$id.outcome))
# 	G <- list()
# 	PHEN <- list(x)
# 	j <- 2
# 	if(any(grepl("U1", traitlist)))
# 	{
# 		u1traits <- gsub("U1_", "", traitlist[grepl("U1", traitlist)]) %>% as.numeric()
# 		message("U1 traits: ", paste(u1traits, collapse=","))
# 		G[[1]] <- lapply(u1traits, function(x) {
# 			return(gu1[[x]][,-1])
# 		}) %>% do.call(cbind, .)
# 		for(i in u1traits)
# 		{
# 			PHEN[[j]] <- u1[,i]
# 			j <- j + 1
# 		}
# 	}
# 	if(any(grepl("U2", traitlist)))
# 	{
# 		u2traits <- gsub("U2_", "", traitlist[grepl("U2", traitlist)]) %>% as.numeric()
# 		message("U2 traits: ", paste(u2traits, collapse=","))
# 		G[[2]] <- lapply(u2traits, function(x) {
# 			return(gu2[[x]][,-1])
# 		}) %>% do.call(cbind, .)
# 		for(i in u2traits)
# 		{
# 			PHEN[[j]] <- u2[,i]
# 			j <- j + 1
# 		}
# 	}
# 	if(any(grepl("U3", traitlist)))
# 	{
# 		u3traits <- gsub("U3_", "", traitlist[grepl("U3", traitlist)]) %>% as.numeric()
# 		message("U3 traits: ", paste(u3traits, collapse=","))
# 		G[[3]] <- lapply(u3traits, function(x) {
# 			return(gu3[[x]])
# 		}) %>% do.call(cbind, .)
# 		for(i in u3traits)
# 		{
# 			PHEN[[j]] <- u3[,i]
# 			j <- j + 1
# 		}
# 	}
# 	if(any(grepl("U4", traitlist)))
# 	{
# 		u4traits <- gsub("U4_", "", traitlist[grepl("U4", traitlist)]) %>% as.numeric()
# 		message("U4 traits: ", paste(u4traits, collapse=","))
# 		G[[4]] <- lapply(u4traits, function(x) {
# 			return(gx)
# 		}) %>% do.call(cbind, .)
# 		for(i in u4traits)
# 		{
# 			PHEN[[j]] <- u4[,i]
# 			j <- j + 1
# 		}
# 	}
# 	G[[5]] <- gx
# 	G <- do.call(cbind, G)
# 	traitlist <- c("X", traitlist)
# 	names(PHEN) <- traitlist

# 	out$mvres <- try({
# 		mvdat <- make_mvdat(PHEN, y, G)
# 		mvres <- mv_multiple(mvdat)
# 		mvres$result$exposure <- traitlist
# 		mvres
# 	})
# 	## get effects
# 	u1x <- list()
# 	u1y <- list()
# 	for(i in 1:nu1)
# 	{
# 		u1x[[i]] <- get_effs(u1[,i], x, gu1[[i]], paste0("U1_", i), "X")
# 		u1y[[i]] <- get_effs(u1[,i], y, gu1[[i]], paste0("U1_", i), "Y")
# 		u1x[[i]]$SNP <- u1y[[i]]$SNP <- c(i, paste0("u1_",1:(ngu1-1)))
# 		u1x[[i]]$effect_allele.exposure <- u1y[[i]]$effect_allele.exposure <- "A"
# 		u1x[[i]]$other_allele.exposure <- u1y[[i]]$other_allele.exposure <- "G"
# 		u1x[[i]]$effect_allele.outcome <- u1y[[i]]$effect_allele.outcome <- "A"
# 		u1x[[i]]$other_allele.outcome <- u1y[[i]]$other_allele.outcome <- "G"
# 		u1x[[i]]$eaf.exposure <- u1y[[i]]$eaf.exposure <- 0.5
# 		u1x[[i]]$eaf.outcome <- u1y[[i]]$eaf.outcome <- 0.5
# 	}

# 	u2x <- list()
# 	u2y <- list()
# 	for(i in 1:nu2)
# 	{
# 		u2x[[i]] <- get_effs(u2[,i], x, gu2[[i]], paste0("U2_", i), "X")
# 		u2y[[i]] <- get_effs(u2[,i], y, gu2[[i]], paste0("U2_", i), "Y")
# 		u2x[[i]]$SNP <- u2y[[i]]$SNP <- c(i+nu1, paste0("u2_",1:(ngu2-1)))
# 		u2x[[i]]$effect_allele.exposure <- u2y[[i]]$effect_allele.exposure <- "A"
# 		u2x[[i]]$other_allele.exposure <- u2y[[i]]$other_allele.exposure <- "G"
# 		u2x[[i]]$effect_allele.outcome <- u2y[[i]]$effect_allele.outcome <- "A"
# 		u2x[[i]]$other_allele.outcome <- u2y[[i]]$other_allele.outcome <- "G"
# 		u2x[[i]]$eaf.exposure <- u2y[[i]]$eaf.exposure <- 0.5
# 		u2x[[i]]$eaf.outcome <- u2y[[i]]$eaf.outcome <- 0.5
# 	}

# 	u3x <- list()
# 	u3y <- list()
# 	for(i in 1:nu3)
# 	{
# 		u3x[[i]] <- get_effs(u3[,i], x, gu3[[i]], paste0("U3_", i), "X")
# 		u3y[[i]] <- get_effs(u3[,i], y, gu3[[i]], paste0("U3_", i), "Y")
# 		u3x[[i]]$SNP <- u3y[[i]]$SNP <- c(i+nu1, paste0("u3_",1:(ngu3-1)))
# 		u3x[[i]]$effect_allele.exposure <- u3y[[i]]$effect_allele.exposure <- "A"
# 		u3x[[i]]$other_allele.exposure <- u3y[[i]]$other_allele.exposure <- "G"
# 		u3x[[i]]$effect_allele.outcome <- u3y[[i]]$effect_allele.outcome <- "A"
# 		u3x[[i]]$other_allele.outcome <- u3y[[i]]$other_allele.outcome <- "G"
# 		u3x[[i]]$eaf.exposure <- u3y[[i]]$eaf.exposure <- 0.5
# 		u3x[[i]]$eaf.outcome <- u3y[[i]]$eaf.outcome <- 0.5
# 	}

# 	u4x <- list()
# 	u4y <- list()
# 	for(i in 1:nu4)
# 	{
# 		u4x[[i]] <- get_effs(u4[,i], x, gx[,1:ngu4], paste0("U4_", i), "X")
# 		u4y[[i]] <- get_effs(u4[,i], y, gx[,1:ngu4], paste0("U4_", i), "Y")
# 		u4x[[i]]$SNP <- u4y[[i]]$SNP <- c(i+nu1, paste0("u4_",1:(ngu4-1)))
# 		u4x[[i]]$effect_allele.exposure <- u4y[[i]]$effect_allele.exposure <- "A"
# 		u4x[[i]]$other_allele.exposure <- u4y[[i]]$other_allele.exposure <- "G"
# 		u4x[[i]]$effect_allele.outcome <- u4y[[i]]$effect_allele.outcome <- "A"
# 		u4x[[i]]$other_allele.outcome <- u4y[[i]]$other_allele.outcome <- "G"
# 		u4x[[i]]$eaf.exposure <- u4y[[i]]$eaf.exposure <- 0.5
# 		u4x[[i]]$eaf.outcome <- u4y[[i]]$eaf.outcome <- 0.5
# 	}

# 	out$candidate_instruments <- subset(rbind(bind_rows(u1x), bind_rows(u2x), bind_rows(u3x)), select=c(
# 		SNP, beta.exposure, se.exposure, id.exposure, exposure, effect_allele.exposure, other_allele.exposure, eaf.exposure, pval.exposure
# 	))
# 	out2 <- subset(out$search, sig)
# 	out$candidate_instruments <- group_by(out$candidate_instruments, id.exposure) %>%
# 		do({
# 			x <- .
# 			y <- subset(out2, id.outcome == x$id.exposure[1])
# 			x <- subset(x, !SNP %in% y$SNP)
# 			x
# 		})

# 	out$candidate_outcome <- subset(rbind(bind_rows(u1y), bind_rows(u2y), bind_rows(u3y)), select=c(
# 		SNP, beta.outcome, se.outcome, id.outcome, outcome, effect_allele.outcome, other_allele.outcome, eaf.outcome, pval.outcome
# 	))

# 	# out$candidate_outcome_dat <- suppressMessages(harmonise_data(out$candidate_instruments, out$candidate_outcome))
# 	out$candidate_outcome_dat <- bind_rows(bind_rows(u1y), bind_rows(u2y), bind_rows(u3y))
# 	out$candidate_outcome_mr <- suppressMessages(mr(out$candidate_outcome_dat, method_list="mr_ivw"))

# 	out$candidate_exposure <- subset(rbind(bind_rows(u1x), bind_rows(u2x), bind_rows(u3x)), select=c(
# 		SNP, beta.outcome, se.outcome, id.outcome, outcome, effect_allele.outcome, other_allele.outcome, eaf.outcome, pval.outcome
# 	))

# 	# out$candidate_exposure_dat <- suppressMessages(harmonise_data(out$candidate_instruments, out$candidate_exposure))
# 	out$candidate_exposure_dat <- bind_rows(bind_rows(u1x), bind_rows(u2x), bind_rows(u3x))
# 	out$candidate_exposure_mr <- suppressMessages(mr(out$candidate_exposure_dat, method_list="mr_ivw"))

# 	out$simulation <- list()
# 	out$simulation$no_outlier_flag <- no_outlier_flag
# 	out$simulation$nu1 <- nu1
# 	out$simulation$nu2 <- nu2
# 	out$simulation$nu3 <- nu3
# 	out$simulation$bxu3 <- bxu3
# 	out$simulation$bu3y <- bu3y
# 	out$simulation$outliers_known <- outliers_known
# 	out$simulation$bxy <- bxy

# 	if(debug)
# 	{
# 		out$phen <- list(
# 			y=y, x=x, u1=u1, u2=u2, u3=u3, u4=u4
# 		)
# 	}
# 	return(out)
# }


#' Uses an ARTMAP network to classify the given input data.
#'
#' This function uses an ARTMAP network to classify the given input data with
#' the specified vigilance parameter. Each sample of the data is presented to
#' the network, which classifies each sample. The function returns the
#' classification of each sample. If the classification of the sample requires
#' that a new category be created, the class for that sample is set to -1.
#' @title ARTMAP_Classify
#' @param artmap_network The trained ARTMAP network. It should be created with ARTMAP_Create_Network() and trained with ARTMAP_Learn()
#' @param data Classification data to be presented to the network. It is a matrix of size NumFeatures-by-NumSamples.
#' @return Vector of size NumSamples that holds the class in which the ARTMAP network placed each sample.
#' @export
ARTMAP_Classify=function(artmap_network, data)
{
    # Make sure the user specifies the input parameters.
  if(nargs() != 2)
  {
    stop('You must specify both input parameters.');
  }

  # Make sure that the data is appropriate for the given network.
  numFeatures = ncol(data);
  numSamples = nrow(data);

  if(numFeatures != artmap_network$numFeatures)
  {
    stop('The data does not contain the same number of features as the network.');
  }

  # Make sure the vigilance is within the (0, 1] range.
  if((artmap_network$vigilance <= 0) | (artmap_network$vigilance > 1))
  {
    stop('The vigilance must be within the range (0, 1].');
  }

  # Set up the return variables.
  classification = matrix(0,numSamples,1);

  # Classify and learn on each sample.
  for (sampleNumber in 1:numSamples)
  {
    # Get the current data sample.
    currentData = data[sampleNumber,];

    # Activate the categories for this sample.
    bias = artmap_network$bias;
    categoryActivation = ART_Activate_Categories(currentData, artmap_network$weight, bias);

    # Rank the activations in order from highest to lowest.
    # This will allow us easier access to step through the categories.
    auxList = sort(-categoryActivation,index.return=TRUE);
    sortedActivations=auxList[[1]];
    sortedCategories=auxList[[2]];
    # Go through each category in the sorted list looking for the best match.
    resonance = 0;
    match = 0;
    numSortedCategories = length(sortedCategories);
    currentSortedIndex = 1;
    while(!resonance)
    {

      # If there are no categories, we return a -1.
      if(numSortedCategories == 0)
      {
        classification[sampleNumber,1] = -1;
        resonance = 1;
        break;
      }

      # Get the current category based on the sorted index.
      currentCategory = sortedCategories[currentSortedIndex];

      # Get the current weight vector from the sorted category list.
      currentWeightVector = artmap_network$weight[currentCategory,];

      # Calculate the match given the current data sample and weight vector.
      match = ART_Calculate_Match(currentData, currentWeightVector);

      # Check to see if the match is less than the vigilance.
      if(match < artmap_network$vigilance)
      {
        # If so, choose the next category in the sorted category list.
        # If the current category is the last in the list, set the
        # category for the return value as -1 and induce resonance.
        if(currentSortedIndex == numSortedCategories)
        {
          classification[sampleNumber,] = -1;
          resonance = 1;
        }
        else
        {
          currentSortedIndex = currentSortedIndex + 1;
        }
      }
      else
      {
        # Otherwise, the current category codes the input.
        # Therefore, we should induce resonance.
        classification[sampleNumber,] = artmap_network$mapField[currentCategory,];
        resonance = 1;
      }
    }
  }
  return (classification);
}


score <- function(input) {
    var0 <- subroutine0(input)
    var1 <- subroutine1(input)
    var2 <- subroutine2(input)
    var3 <- subroutine3(input)
    var4 <- subroutine4(input)
    var5 <- subroutine5(input)
    var6 <- subroutine6(input)
    var7 <- subroutine7(input)
    var8 <- subroutine8(input)
    var9 <- subroutine9(input)
    var10 <- subroutine10(input)
    var11 <- subroutine11(input)
    var12 <- subroutine12(input)
    var13 <- subroutine13(input)
    var14 <- subroutine14(input)
    var15 <- subroutine15(input)
    var16 <- subroutine16(input)
    var17 <- subroutine17(input)
    var18 <- subroutine18(input)
    var19 <- subroutine19(input)
    var20 <- subroutine20(input)
    var21 <- subroutine21(input)
    var22 <- subroutine22(input)
    var23 <- subroutine23(input)
    var24 <- subroutine24(input)
    var25 <- subroutine25(input)
    var26 <- subroutine26(input)
    var27 <- subroutine27(input)
    var28 <- subroutine28(input)
    var29 <- subroutine29(input)
    var30 <- subroutine30(input)
    var31 <- subroutine31(input)
    var32 <- subroutine32(input)
    var33 <- subroutine33(input)
    var34 <- (var21) * (-29.893441720221567)
    return(c((((((((((((((((((((((-0.10040896360166912) + ((var0) * (-0.0))) + ((var1) * (-0.0))) + ((var2) * (-0.41839590533727394))) + ((var3) * (-0.0))) + ((var4) * (-0.04143676373258231))) + ((var5) * (-0.41839590533727394))) + ((var6) * (-0.41839590533727394))) + ((var7) * (-0.0))) + ((var8) * (-0.09370933375375942))) + ((var9) * (-0.021427693031370827))) + ((var10) * (-0.19906272901896646))) + ((var11) * (-0.41839590533727394))) + ((var12) * (0.01790941942476885))) + ((var13) * (0.19838472644676905))) + ((var14) * (0.41839590533727394))) + ((var15) * (0.0))) + ((var16) * (0.41839590533727394))) + ((var17) * (0.13934237366514107))) + ((var18) * (0.0))) + ((var19) * (0.41839590533727394))) + ((var20) * (0.41839590533727394)), ((((((((((((((((((((((-0.18554309515982967) + ((var21) * (-0.06950455855790763))) + ((var22) * (-0.0))) + ((var23) * (-0.04843045931251089))) + ((var24) * (-0.0))) + ((var25) * (-0.3538789357824905))) + ((var26) * (-0.0609756100198299))) + ((var27) * (-0.11288644645628641))) + ((var28) * (-0.3538789357824905))) + ((var29) * (-0.30936738135098624))) + ((var30) * (-0.3538789357824905))) + ((var31) * (-0.0))) + ((var32) * (-0.08889946907833211))) + ((var33) * (-0.0))) + ((var12) * (0.14642472878044094))) + ((var13) * (0.11750372941413528))) + ((var14) * (0.3538789357824905))) + ((var15) * (0.3538789357824905))) + ((var16) * (0.0))) + ((var17) * (0.0722565307987864))) + ((var18) * (0.3538789357824905))) + ((var19) * (0.3538789357824905))) + ((var20) * (0.0)), (((((((((((((((((((((((((0.5039648533253523) + (var34)) + ((var22) * (-29.893441720221567))) + ((var23) * (-0.0))) + ((var24) * (-23.66820011477251))) + ((var25) * (-0.0))) + ((var26) * (-0.0))) + ((var27) * (-0.0))) + ((var28) * (-7.128872495868185))) + ((var29) * (-19.888600728398526))) + ((var30) * (-0.4282859091720985))) + ((var31) * (-5.683580020209366))) + ((var32) * (-0.0))) + ((var33) * (-29.893441720221567))) + ((var0) * (15.40619227503478))) + ((var1) * (11.497905273164342))) + ((var2) * (0.0))) + ((var3) * (29.893441720221567))) + ((var4) * (29.893441720221567))) + ((var5) * (0.0))) + ((var6) * (0.0))) + ((var7) * (29.893441720221567))) + ((var8) * (29.893441720221567))) + ((var9) * (0.0))) + ((var10) * (0.0))) + ((var11) * (0.0))))
}
subroutine0 <- function(input) {
    var0 <- (0) - (0.25)
    return(exp((var0) * ((((((5.4) - (input[1])) ^ (2)) + (((3.0) - (input[2])) ^ (2))) + (((4.5) - (input[3])) ^ (2))) + (((1.5) - (input[4])) ^ (2)))))
}
subroutine1 <- function(input) {
    var0 <- (0) - (0.25)
    return(exp((var0) * ((((((6.2) - (input[1])) ^ (2)) + (((2.2) - (input[2])) ^ (2))) + (((4.5) - (input[3])) ^ (2))) + (((1.5) - (input[4])) ^ (2)))))
}
subroutine2 <- function(input) {
    var0 <- (0) - (0.25)
    return(exp((var0) * ((((((5.0) - (input[1])) ^ (2)) + (((2.3) - (input[2])) ^ (2))) + (((3.3) - (input[3])) ^ (2))) + (((1.0) - (input[4])) ^ (2)))))
}
subroutine3 <- function(input) {
    var0 <- (0) - (0.25)
    return(exp((var0) * ((((((5.9) - (input[1])) ^ (2)) + (((3.2) - (input[2])) ^ (2))) + (((4.8) - (input[3])) ^ (2))) + (((1.8) - (input[4])) ^ (2)))))
}
subroutine4 <- function(input) {
    var0 <- (0) - (0.25)
    return(exp((var0) * ((((((6.7) - (input[1])) ^ (2)) + (((3.0) - (input[2])) ^ (2))) + (((5.0) - (input[3])) ^ (2))) + (((1.7) - (input[4])) ^ (2)))))
}
subroutine5 <- function(input) {
    var0 <- (0) - (0.25)
    return(exp((var0) * ((((((7.0) - (input[1])) ^ (2)) + (((3.2) - (input[2])) ^ (2))) + (((4.7) - (input[3])) ^ (2))) + (((1.4) - (input[4])) ^ (2)))))
}
subroutine6 <- function(input) {
    var0 <- (0) - (0.25)
    return(exp((var0) * ((((((4.9) - (input[1])) ^ (2)) + (((2.4) - (input[2])) ^ (2))) + (((3.3) - (input[3])) ^ (2))) + (((1.0) - (input[4])) ^ (2)))))
}
subroutine7 <- function(input) {
    var0 <- (0) - (0.25)
    return(exp((var0) * ((((((6.3) - (input[1])) ^ (2)) + (((2.5) - (input[2])) ^ (2))) + (((4.9) - (input[3])) ^ (2))) + (((1.5) - (input[4])) ^ (2)))))
}
subroutine8 <- function(input) {
    var0 <- (0) - (0.25)
    return(exp((var0) * ((((((6.0) - (input[1])) ^ (2)) + (((2.7) - (input[2])) ^ (2))) + (((5.1) - (input[3])) ^ (2))) + (((1.6) - (input[4])) ^ (2)))))
}
subroutine9 <- function(input) {
    var0 <- (0) - (0.25)
    return(exp((var0) * ((((((5.7) - (input[1])) ^ (2)) + (((2.6) - (input[2])) ^ (2))) + (((3.5) - (input[3])) ^ (2))) + (((1.0) - (input[4])) ^ (2)))))
}
subroutine10 <- function(input) {
    var0 <- (0) - (0.25)
    return(exp((var0) * ((((((6.9) - (input[1])) ^ (2)) + (((3.1) - (input[2])) ^ (2))) + (((4.9) - (input[3])) ^ (2))) + (((1.5) - (input[4])) ^ (2)))))
}
subroutine11 <- function(input) {
    var0 <- (0) - (0.25)
    return(exp((var0) * ((((((5.1) - (input[1])) ^ (2)) + (((2.5) - (input[2])) ^ (2))) + (((3.0) - (input[3])) ^ (2))) + (((1.1) - (input[4])) ^ (2)))))
}
subroutine12 <- function(input) {
    var0 <- (0) - (0.25)
    return(exp((var0) * ((((((5.1) - (input[1])) ^ (2)) + (((3.8) - (input[2])) ^ (2))) + (((1.9) - (input[3])) ^ (2))) + (((0.4) - (input[4])) ^ (2)))))
}
subroutine13 <- function(input) {
    var0 <- (0) - (0.25)
    return(exp((var0) * ((((((4.4) - (input[1])) ^ (2)) + (((2.9) - (input[2])) ^ (2))) + (((1.4) - (input[3])) ^ (2))) + (((0.2) - (input[4])) ^ (2)))))
}
subroutine14 <- function(input) {
    var0 <- (0) - (0.25)
    return(exp((var0) * ((((((5.7) - (input[1])) ^ (2)) + (((4.4) - (input[2])) ^ (2))) + (((1.5) - (input[3])) ^ (2))) + (((0.4) - (input[4])) ^ (2)))))
}
subroutine15 <- function(input) {
    var0 <- (0) - (0.25)
    return(exp((var0) * ((((((5.8) - (input[1])) ^ (2)) + (((4.0) - (input[2])) ^ (2))) + (((1.2) - (input[3])) ^ (2))) + (((0.2) - (input[4])) ^ (2)))))
}
subroutine16 <- function(input) {
    var0 <- (0) - (0.25)
    return(exp((var0) * ((((((5.1) - (input[1])) ^ (2)) + (((3.3) - (input[2])) ^ (2))) + (((1.7) - (input[3])) ^ (2))) + (((0.5) - (input[4])) ^ (2)))))
}
subroutine17 <- function(input) {
    var0 <- (0) - (0.25)
    return(exp((var0) * ((((((5.7) - (input[1])) ^ (2)) + (((3.8) - (input[2])) ^ (2))) + (((1.7) - (input[3])) ^ (2))) + (((0.3) - (input[4])) ^ (2)))))
}
subroutine18 <- function(input) {
    var0 <- (0) - (0.25)
    return(exp((var0) * ((((((4.3) - (input[1])) ^ (2)) + (((3.0) - (input[2])) ^ (2))) + (((1.1) - (input[3])) ^ (2))) + (((0.1) - (input[4])) ^ (2)))))
}
subroutine19 <- function(input) {
    var0 <- (0) - (0.25)
    return(exp((var0) * ((((((4.5) - (input[1])) ^ (2)) + (((2.3) - (input[2])) ^ (2))) + (((1.3) - (input[3])) ^ (2))) + (((0.3) - (input[4])) ^ (2)))))
}
subroutine20 <- function(input) {
    var0 <- (0) - (0.25)
    return(exp((var0) * ((((((4.8) - (input[1])) ^ (2)) + (((3.4) - (input[2])) ^ (2))) + (((1.9) - (input[3])) ^ (2))) + (((0.2) - (input[4])) ^ (2)))))
}
subroutine21 <- function(input) {
    var0 <- (0) - (0.25)
    return(exp((var0) * ((((((6.0) - (input[1])) ^ (2)) + (((3.0) - (input[2])) ^ (2))) + (((4.8) - (input[3])) ^ (2))) + (((1.8) - (input[4])) ^ (2)))))
}
subroutine22 <- function(input) {
    var0 <- (0) - (0.25)
    return(exp((var0) * ((((((6.3) - (input[1])) ^ (2)) + (((2.8) - (input[2])) ^ (2))) + (((5.1) - (input[3])) ^ (2))) + (((1.5) - (input[4])) ^ (2)))))
}
subroutine23 <- function(input) {
    var0 <- (0) - (0.25)
    return(exp((var0) * ((((((5.8) - (input[1])) ^ (2)) + (((2.8) - (input[2])) ^ (2))) + (((5.1) - (input[3])) ^ (2))) + (((2.4) - (input[4])) ^ (2)))))
}
subroutine24 <- function(input) {
    var0 <- (0) - (0.25)
    return(exp((var0) * ((((((6.1) - (input[1])) ^ (2)) + (((3.0) - (input[2])) ^ (2))) + (((4.9) - (input[3])) ^ (2))) + (((1.8) - (input[4])) ^ (2)))))
}
subroutine25 <- function(input) {
    var0 <- (0) - (0.25)
    return(exp((var0) * ((((((7.7) - (input[1])) ^ (2)) + (((2.6) - (input[2])) ^ (2))) + (((6.9) - (input[3])) ^ (2))) + (((2.3) - (input[4])) ^ (2)))))
}
subroutine26 <- function(input) {
    var0 <- (0) - (0.25)
    return(exp((var0) * ((((((6.9) - (input[1])) ^ (2)) + (((3.1) - (input[2])) ^ (2))) + (((5.1) - (input[3])) ^ (2))) + (((2.3) - (input[4])) ^ (2)))))
}
subroutine27 <- function(input) {
    var0 <- (0) - (0.25)
    return(exp((var0) * ((((((6.3) - (input[1])) ^ (2)) + (((3.3) - (input[2])) ^ (2))) + (((6.0) - (input[3])) ^ (2))) + (((2.5) - (input[4])) ^ (2)))))
}
subroutine28 <- function(input) {
    var0 <- (0) - (0.25)
    return(exp((var0) * ((((((4.9) - (input[1])) ^ (2)) + (((2.5) - (input[2])) ^ (2))) + (((4.5) - (input[3])) ^ (2))) + (((1.7) - (input[4])) ^ (2)))))
}
subroutine29 <- function(input) {
    var0 <- (0) - (0.25)
    return(exp((var0) * ((((((6.0) - (input[1])) ^ (2)) + (((2.2) - (input[2])) ^ (2))) + (((5.0) - (input[3])) ^ (2))) + (((1.5) - (input[4])) ^ (2)))))
}
subroutine30 <- function(input) {
    var0 <- (0) - (0.25)
    return(exp((var0) * ((((((7.9) - (input[1])) ^ (2)) + (((3.8) - (input[2])) ^ (2))) + (((6.4) - (input[3])) ^ (2))) + (((2.0) - (input[4])) ^ (2)))))
}
subroutine31 <- function(input) {
    var0 <- (0) - (0.25)
    return(exp((var0) * ((((((7.2) - (input[1])) ^ (2)) + (((3.0) - (input[2])) ^ (2))) + (((5.8) - (input[3])) ^ (2))) + (((1.6) - (input[4])) ^ (2)))))
}
subroutine32 <- function(input) {
    var0 <- (0) - (0.25)
    return(exp((var0) * ((((((7.7) - (input[1])) ^ (2)) + (((3.8) - (input[2])) ^ (2))) + (((6.7) - (input[3])) ^ (2))) + (((2.2) - (input[4])) ^ (2)))))
}
subroutine33 <- function(input) {
    var0 <- (0) - (0.25)
    return(exp((var0) * ((((((6.2) - (input[1])) ^ (2)) + (((2.8) - (input[2])) ^ (2))) + (((4.8) - (input[3])) ^ (2))) + (((1.8) - (input[4])) ^ (2)))))
}


#' Calculate topological indices for ecological networks. The mean trophic level
#' and Omnivory and the Level of omnivory are calculated with the function [NetIndices::TrophInd()]
#'
#' @param ig vector of igraph objects
#' @param ncores number of cores used to compute in parallel, if 0 sequential processing is used.
#'
#' @return a data.frame with the following fields:
#'
#'  \item{Size:}{Number of species}
#'  \item{Top:}{Number of top predator species}
#'  \item{Basal:}{Number of basal especies}
#'  \item{Omnivory:}{ Proportion of omnivorous species }
#'  \item{Links:}{ number of interactions}
#'  \item{LD:}{ linkage density}
#'  \item{Connectance:}{ directed Connectance}
#'  \item{PathLength:}{ average path length}
#'  \item{Clustering:}{ clustering coeficient}
#'  \item{Cannib:}{ number of cannibalistic species}
#'  \item{TLmean:}{ mean trophic level}
#'  \item{TLmax:}{ maximum trophic level}
#'  \item{LOmnivory:} { Level of omnivory, quantiﬁes mean of the variety in trophic levels of the preys of a consumer}
#'  \item{Components:}{ number of weakly connected components}
#'  \item{Vulnerability:}{ mean of number of consumers per prey}
#'  \item{VulSD:}{ the standard deviation of normalized Vulnerability}
#'  \item{Generality:}{ mean number of prey per consumer}
#'  \item{GenSD:}{ the standard deviation of normalized Generality}
#'
#'
#' @export
#' @importFrom NetIndices TrophInd
#' @import igraph
#' @importFrom foreach foreach %dopar%
#' @importFrom doFuture registerDoFuture
#' @importFrom future sequential multiprocess
#'
#' @aliases calcTopologicalIndices
#'
#' @examples
#'
#' calc_topological_indices(netData)
#'
#' # Generate a test network
#'
#' g <-   graph_from_literal( 1 -+ 4 -+ 7,2 -+ 5 -+7, 3-+6-+7, 7-+7, 4+-3, simplify = FALSE)
#'
#' calc_topological_indices(g)

calc_topological_indices <- function(ig,ncores=0){

  if(inherits(ig,"igraph")) {
    ig <- list(ig)
  } else if(class(ig[[1]])!="igraph") {
    stop("parameter ig must be an igraph object")
  }

  registerDoFuture()
  if(ncores) {
    cn <- future::availableCores()
    if(ncores>cn)
      ncores <- cn
    future::plan(multiprocess, workers=ncores)
    on.exit(future::plan(sequential))
  } else {
    future::plan(sequential)
  }

  df <-  foreach(g=ig,.combine='rbind',.inorder=FALSE,.packages=c('igraph','NetIndices')) %dopar%
    {

    deg <- degree(igraph::simplify(g), mode="out") # calculate the out-degree: the number of predators

    V(g)$outdegree <-  deg

    nTop <- length(V(g)[outdegree==0]) # Top predators do not have predators

    deg <- degree(g, mode="in") # calculate the in-degree: the number of preys

    V(g)$indegree <-  deg

    nBasal <- length(V(g)[indegree==0]) # Basal species do not have preys

    #vcount(g)-nTop-nBasal

    size <- vcount(g)

    links <- ecount(g)

    linkDen <- links/size          # Linkage density

    conn <- links/size^2           # Connectance

    pathLength <- mean_distance(g)   # Average path length

    clusCoef <- transitivity(g, type = "global")

    cannib <- sum(which_loop(g))

    a <- get.adjacency(g,sparse=F)

    tl <- TrophInd(a)  # Calculate the trophic level

    omn <- sum(round(tl$OI,5)>0)/size        # Omnivory proportion

    lomn <- mean(tl$OI)                      # Level of omnivory

    vulnerability <- links / (size - nTop)    # l/(nb + ni)
    generality    <- links / (size - nBasal)  # l/(nt + ni)
    vulSD         <- sd(apply(a, 1, sum)*1/linkDen)
    genSD         <- sd(apply(a, 2, sum)*1/linkDen)

    data.frame(Size=size,Top=nTop,Basal=nBasal,Omnivory=omn,Links=links, LD=linkDen,Connectance=conn,PathLength=pathLength,
               Clustering=clusCoef, Cannib=cannib, TLmean=mean(tl$TL),TLmax=max(tl$TL),LOmnivory=lomn,Components=components(g)$no,
               Vulnerability=vulnerability,VulSD=vulSD,Generality=generality,GenSD=genSD)
  }
return(df)
}

#' @export
calcTopologicalIndices <- function(ig,ncores=0){
    calc_topological_indices(ig,ncores)
}

#' Calculate the incoherence index of a food web
#'
#' The incoherence index is based in how the species fit in discrete trophic levels
#' when Q is closer to 0 more coherent and stable is a food web, and the less omnivory it has.
#' It calculates the trophic level using the package NetIndices.
#'
#' Based on:
#'
#' @references Johnson, S., Domínguez-García, V., Donetti, L., & Muñoz, M. A. (2014). Trophic coherence determines food-web stability. Proceedings of the National Academy of Sciences , 111(50), 17923–17928. https://doi.org/10.1073/pnas.1409077111
#'
#' @references Johnson, S., & Jones, N. S. (2017). Looplessness in networks is linked to trophic coherence. Proceedings of the National Academy of Sciences, 114(22), 5618–5623. https://doi.org/10.1073/pnas.1613786114
#'
#'
#' @param ig an igraph object or a list of igraph objects
#' @param ncores number of cores used to compute in parallel, if 0 sequential processing is used.
#'
#' @return a data.frame with the following fields
#'
#'  \item{Q}{incoherence (0=coherent)}
#'  \item{rQ}{ratio of Q with expected Q under null expectation of a random network given N=nodes L=links B=basal nodes Lb=basal links}
#'  \item{mTI}{mean trophic level}
#'  \item{rTI}{ratio of mTI with expected TI under the same null model expectation than Q}
#
#' @aliases calcIncoherence
#'
#' @export
#'
#' @examples
#'
#' calc_incoherence(netData[[1]])
#'
#'
#' @importFrom NetIndices TrophInd
#' @importFrom igraph     V degree get.adjacency vcount ecount
#' @importFrom foreach foreach %dopar%
#' @importFrom doFuture registerDoFuture
#' @importFrom future sequential multiprocess

calc_incoherence <- function(ig,ncores=0) {

  if(inherits(ig,"igraph")) {
    ig <- list(ig)
  } else if(class(ig[[1]])!="igraph") {
    stop("parameter ig must be an igraph object")
  }

  registerDoFuture()
  if(ncores) {
    cn <- future::availableCores()
    if(ncores>cn)
      ncores <- cn
    future::plan(multiprocess, workers=ncores)
    on.exit(future::plan(sequential))
  } else {
    future::plan(sequential)
  }

  # if(!is.null(ncores)) {
  #   cn <-parallel::detectCores()
  #   if(cn>ncores)
  #     cn <- ncores
  #   else
  #     cn <- cn-1
  #
  #   # cl <- makeCluster(cn,outfile="foreach.log") # Logfile to debug
  #   cl <- parallel::makeCluster(cn)
  #   doParallel::registerDoParallel(cl)
  #   on.exit(parallel::stopCluster(cl))
  # } else {
  #   foreach::registerDoSEQ()
  # }

  df <-  foreach(g=ig,.combine='rbind',.inorder=FALSE,.packages=c('igraph','NetIndices')) %dopar%
  {
    ti<-TrophInd(get.adjacency(g,sparse=FALSE))
    v <- ti$TL
    z <- round(outer(v,v,'-'),8);
    A <- get.adjacency(g,sparse = FALSE)
    xx <- A>0
    x <- (A*t(z))[xx]
    Q <- round(sqrt(sum((x-1)^2)/ecount(g) ),8)

    basal <- which(round(v,8)==1)
    bedges <- sum(degree(g,basal,mode='out'))
    mTI <- mean(v)
    mK <- mean(degree(g,mode='out'))
    eTI <- 1+(1-length(basal)/vcount(g))*ecount(g)/bedges
    eQ <- sqrt(ecount(g)/bedges-1)
    data.frame(Q=Q,rQ=Q/eQ,mTI=mTI,rTI=mTI/eTI)
  }
  return(df)
}

#' @export
calcIncoherence <- function(ig,ncores=0){
  calc_incoherence(ig,ncores)
  }

#' Calculation of Modularity and Small-world-ness z-scores
#'
#' The function calculates modularity, number of groups and small-world-ness z-scores and 99\% CI intervals
#' using as null model the list of networks in the nullDist parameters. Modularity is calculated using the [igraph::cluster_spinglass()]
#' if the parameter weights is NULL the atribute "weigths" is used, or if it has the name of an network attribute, that is used as a weigth
#' to build the modules, when this parameter is NA then no weigth is used.
#' Only works for one component networks
#'
#' @references
#' Marina, T. I., Saravia, L. A., Cordone, G., Salinas, V., Doyle, S. R., & Momo, F. R. (2018). Architecture of marine food webs: To be or not be a ‘small-world.’ PLoS ONE, 13(5), 1–13. https://doi.org/10.1371/journal.pone.0198217
#'
#' @param g  igraph object
#' @param nullDist list of igraph object with the null model simulations
#' @param sLevel significance level to calculate CI (two tails)
#' @param ncores number of cores to use paralell computation, if 0 sequential processing is used.
#' @param weights The weights of the edges. Either a numeric vector or NULL or NA. If it is null and the input graph has a ‘weight’ edge attribute
#'                then that will be used. If NULL and no such attribute is present then the edges will have equal weights.
#'                Set this to NA if the graph was a ‘weight’ edge attribute, but you don't want to use it for community detection.
#'
#'
#' @return a list with two data frames: one with indices z-scores and CI
#'
#'  \item{Clustering}{ Clustering coefficient, measures the average fraction of pairs of neighbors of a node that are also neighbors of each other}
#'  \item{PathLength}{ Mean of the shortest paths between all pair of vertices }
#'  \item{Modularity}{ modularity measures how separated are different groups from each other, the algorithm \code{cluster_spinglass} was used to obtain the groups}
#'  \item{zCC,zCP,zMO}{Z-scores of Clustering,PathLength and Modularity with respect to a random Erdos-Renyi null model}
#'  \item{CClow,CChigh,CPlow,CPhigh,MOlow,MOhigh}{sLevel confidence intervals}
#'  \item{SWness,SWnessCI}{ Small-world-ness and it CI value}
#'  \item{isSW,isSWness}{ Logical variable signalling if the network is Small-world by the method of Marina 2018 or the method of Humprhies & Gurney 2008 }
#'
#'  Another data.frame with the values calculated for the nullDist.
#'
#' @export
#'
#' @importFrom igraph transitivity average.path.length cluster_spinglass
#' @importFrom future.apply future_lapply
#' @importFrom future sequential multiprocess
#'
#' @examples
#' \dontrun{
#' nullg <- generateERbasal(netData[[1]],10)
#' calcModularitySWnessZScore(netData[[1]],nullg)
#' }
#'
calc_modularity_swness_zscore<- function(g, nullDist,sLevel=0.01,ncores=0,weights=NA){

  if(!is_igraph(g))
    stop("Parameter g must be an igraph object")

  t <- calcTopologicalIndices(g)

  if(length(nullDist)<5)
    stop("nullDist: There has to be more than 5 elements in the list")

  if(any(sapply(nullDist, function(g) components(g)$no>1)))
    stop("nullDist: one or more igraph object from nullDist have more than one component")

  # nullDist <- lapply(1:nsim, function (x) {
  #   e <- sample_gnm(t$Size, t$Links, directed = TRUE)
  #
  #   # Check that the ER networks has only one connected component
  #   #
  #   while(components(e)$no>1)
  #     e <- erdos.renyi.game(t$Size, t$Links, type="gnm",directed = TRUE)
  #
  #   return(e) }
  # )

  if(ncores) {
    cn <- future::availableCores()
    if(ncores>cn)
      ncores <- cn
    future::plan(multiprocess, workers=ncores)
    on.exit(future::plan(sequential))
  } else {
    future::plan(sequential)
  }

  ind <- data.frame()

  ind <- future_lapply(1:length(nullDist), function(i)
  {
    m<-cluster_spinglass(nullDist[[i]],weights=weights)
    modl <- m$modularity
    clus.coef <- transitivity(nullDist[[i]], type="Global")
    cha.path  <- average.path.length(nullDist[[i]])
    data.frame(modularity=modl,clus.coef=clus.coef,cha.path=cha.path)
  }, future.seed = TRUE)
  ind <- bind_rows(ind)

  ind$gamma <- t$Clustering/ind$clus.coef
  ind$lambda <- t$PathLength/ind$cha.path
  ind$SWness <- ind$gamma/ind$lambda

  # 99% confidence interval
  #
  sLevel <- sLevel/2
  qSW <- quantile(ind$SWness,c(sLevel,1-sLevel),na.rm = TRUE)
  qmo <- quantile(ind$modularity,c(sLevel,1-sLevel))

  mcc <- mean(ind$clus.coef)
  mcp <- mean(ind$cha.path)

  zcc <- (t$Clustering-mcc)/sd(ind$clus.coef)
  zcp <- (t$PathLength-mcp)/sd(ind$cha.path)
  qcc <- quantile(ind$clus.coef,c(sLevel,1-sLevel),na.rm = TRUE)
  qcp <- quantile(ind$cha.path,c(sLevel,1-sLevel),na.rm = TRUE)

  m<-cluster_spinglass(g, weights=weights)

  zmo <- (m$modularity - mean(ind$modularity))/sd(ind$modularity)

  mSW <- mean(t$Clustering/mcc*mcp/t$PathLength)
  mCI <- 1+(qSW[2]-qSW[1])/2

  isLowInsideCPL <- t$PathLength<=qcp[2]
  isGreaterCC <- t$Clustering>qcc[2]
  isSW       <- (isLowInsideCPL & isGreaterCC)
  isSWness   <- (mSW>mCI)

  return(list(da=data.frame(Clustering=t$Clustering, PathLength= t$PathLength, Modularity=m$modularity,
    zCC=zcc,zCP=zcp,zMO=zmo,
    CClow=qcc[1],CChigh=qcc[2],CPlow=qcp[1],CPhigh=qcp[2],MOlow=qmo[1],MOhigh=qmo[2],
    SWness=mSW,SWnessCI=mCI,
    isSW=isSW,isSWness=isSWness),sims=ind))
}

#' @export
calcModularitySWnessZScore<- function(g, nullDist,sLevel=0.01,ncores=0){
  calc_modularity_swness_zscore(g, nullDist,sLevel,ncores)
}

#' Calculation Small-world-ness z-scores
#'
#' The function calculates small-world-ness z-scores and 99\% CI intervals,
#' using as null model the list of networks in the nullDist parameter.
#'
#' @references
#' Marina, T. I., Saravia, L. A., Cordone, G., Salinas, V., Doyle, S. R., & Momo, F. R. (2018). Architecture of marine food webs: To be or not be a ‘small-world.’ PLoS ONE, 13(5), 1–13. https://doi.org/10.1371/journal.pone.0198217
#'
#' @param g  igraph object
#' @param nullDist list of igraph object with the null model simulations
#' @param sLevel significance level to calculate CI (two tails)
#' @param ncores number of cores to use paralell computation, if 0 sequential processing is used.
#'
#'
#' @return a list with two data frames: one with indices z-scores and CI
#'
#'  \item{Clustering}{ Clustering coefficient, measures the average fraction of pairs of neighbors of a node that are also neighbors of each other}
#'  \item{PathLength}{ Mean of the shortest paths between all pair of vertices }
#'  \item{Modularity}{ modularity measures how separated are different groups from each other, the algorithm \code{cluster_spinglass} was used to obtain the groups}
#'  \item{zCC,zCP,zMO}{Z-scores of Clustering,PathLength and Modularity with respect to a random Erdos-Renyi null model}
#'  \item{CClow,CChigh,CPlow,CPhigh,MOlow,MOhigh}{sLevel confidence intervals}
#'  \item{SWness,SWnessCI}{ Small-world-ness and it CI value}
#'  \item{isSW,isSWness}{ Logical variable signalling if the network is Small-world by the method of Marina 2018 or the method of Humprhies & Gurney 2008 }
#'
#'  Another data.frame with the values calculated for the nullDist.
#'
#' @export
#'
#' @importFrom igraph transitivity average.path.length
#' @importFrom foreach foreach %dopar%
#' @importFrom doFuture registerDoFuture
#' @importFrom future sequential multiprocess
#'
#' @examples
#' \dontrun{
#' nullg <- generateERbasal(netData[[1]],10)
#' calc_swness_zscore(netData[[1]],nullg)
#' }
#'
calc_swness_zscore<- function(g, nullDist,sLevel=0.01,ncores=0,weights=NA){

  if(!is_igraph(g))
    stop("Parameter g must be an igraph object")

  t <- calcTopologicalIndices(g)

  if(length(nullDist)<5)
    stop("nullDist: There has to be more than 5 elements in the list")

  registerDoFuture()
  if(ncores) {
    cn <- future::availableCores()
    if(ncores>cn)
      ncores <- cn
    future::plan(multiprocess, workers=ncores)
    on.exit(future::plan(sequential))
  } else {
    future::plan(sequential)
  }

  ind <- data.frame()

  ind <- foreach(i=1:length(nullDist),.combine='rbind',.inorder=FALSE,.packages='igraph') %dopar%
    {
      clus.coef <- transitivity(nullDist[[i]], type="Global")
      cha.path  <- average.path.length(nullDist[[i]])
      data.frame(clus.coef=clus.coef,cha.path=cha.path)
    }

  ind$gamma <- t$Clustering/ind$clus.coef
  ind$lambda <- t$PathLength/ind$cha.path
  ind$SWness <- ind$gamma/ind$lambda

  # 99% confidence interval
  #
  sLevel <- sLevel/2
  qSW <- quantile(ind$SWness,c(sLevel,1-sLevel),na.rm = TRUE)

  mcc <- mean(ind$clus.coef)
  mcp <- mean(ind$cha.path)

  zcc <- (t$Clustering-mcc)/sd(ind$clus.coef)
  zcp <- (t$PathLength-mcp)/sd(ind$cha.path)
  qcc <- quantile(ind$clus.coef,c(sLevel,1-sLevel),na.rm = TRUE)
  qcp <- quantile(ind$cha.path,c(sLevel,1-sLevel),na.rm = TRUE)

  mSW <- mean(t$Clustering/mcc*mcp/t$PathLength)
  mCI <- 1+(qSW[2]-qSW[1])/2

  isLowInsideCPL <- t$PathLength<=qcp[2]
  isGreaterCC <- t$Clustering>qcc[2]
  isSW       <- (isLowInsideCPL & isGreaterCC)
  isSWness   <- (mSW>mCI)

  return(list(da=data.frame(Clustering=t$Clustering, PathLength= t$PathLength,
                            zCC=zcc,zCP=zcp,
                            CClow=qcc[1],CChigh=qcc[2],CPlow=qcp[1],CPhigh=qcp[2],
                            SWness=mSW,SWnessCI=mCI,
                            isSW=isSW,isSWness=isSWness),sims=ind))
}

#' Calc topological roles among network communities/modules
#'
#' Topological roles characterize species as its roles between communities or modules, we calculate the modules using
#' the [igraph::cluster_spinglass()] function.
#' Topological roles are described by two parameters: the standardized within-module degree \eqn{dz} and the among-module
#' connectivity participation coefficient \eqn{PC}.  The within-module degree is a z-score that measures how well a species is
#' connected to other species within its own module compared with a random graph. The participation coefficient \eqn{PC}
#' estimates the distribution of the links of species among modules. As the community algorithm is stochastic we run it several
#' times and return the repeated runs for both parameters.
#'
#' @references
#'
#' 1. Guimerà, R. & Nunes Amaral, L.A. (2005). Functional cartography of complex metabolic networks. Nature, 433, 895–900
#'
#' 1. Kortsch, S. et al. 2015. Climate change alters the structure of arctic marine food webs due to poleward shifts of boreal generalists. - Proceedings of the Royal Society B: Biological Sciences 282: 20151546. https://doi.org/10.1098/rspb.2015.1546

#'
#' @param g an Igraph object with the network
#' @param nsim  number of simulations with different community
#' @param ncores  number of cores to use paralell computation, if 0 sequential processing is used.
#'
#' @return a  data frame with two numeric fields: within_module_degree, among_module_conn
#'
#' @export
#'
#' @import igraph
#' @importFrom foreach foreach %dopar%
#' @importFrom doFuture registerDoFuture
#' @importFrom future sequential multiprocess
#'
#' @examples
#' #' \dontrun{
#'
#' g <- netData[[2]]
#'
#' tp <- calc_topological_roles(g,nsim=10)
#'
#' }

calc_topological_roles <- function(g,nsim=1000,ncores=0)
{
  if(!is_igraph(g))
    stop("Parameter g must be an igraph object")

  toRol <- data.frame()

  registerDoFuture()
  if(ncores) {
    cn <- future::availableCores()
    if(ncores>cn)
      ncores <- cn
    future::plan(multiprocess, workers=ncores)
    on.exit(future::plan(sequential))
  } else {
    future::plan(sequential)
  }

  toRol <- foreach(idx=1:nsim,.combine='rbind',.inorder=FALSE,.packages='igraph') %dopar%
  {
    # within-module degree
    #
    # Standarized Within module degree z-score
    #
    m<-cluster_spinglass(g,weights = NA)
    spingB.mem<- m$membership

    l<-vector()
    memMod<-vector()

    for (i in 1:vcount(g)){

      sp.in.deg <- V(g)[nei(i, "in")]
      sp.out.deg<- V(g)[nei(i, "out")]
      mem.sp<-spingB.mem[i]
      k<- length(which(spingB.mem[c(sp.in.deg, sp.out.deg)]==mem.sp))
      mem<- which(spingB.mem==mem.sp)

      for (m in 1:length(mem)){
        mem.in.deg <- V(g)[nei(mem[m], "in")]
        mem.out.deg<- V(g)[nei(mem[m], "out")]
        memMod.id<- length(which(spingB.mem[c(mem.in.deg, mem.out.deg)]==mem.sp))
        memMod[m]<- memMod.id
      }

      k.ave<- mean(memMod)
      k.sd<- sd(memMod)
      l[i]<- (k-k.ave)/k.sd
    }

    # among module connectivity
    r<- vector()
    for (i in 1:vcount(g)){

      d<-degree(g)[i]
      sp.in.deg <- V(g)[nei(i, "in")]
      sp.out.deg<- V(g)[nei(i, "out")]
      mod.sp<-table(spingB.mem[c(sp.in.deg, sp.out.deg)])
      mod.no<-as.numeric(names(mod.sp))
      mod<-rep(0, length(unique(spingB.mem)))
      mod[mod.no]<-c(mod.sp)
      r[i]<- 1-(sum((mod/d)^2))
    }
    return(data.frame(node=1:vcount(g),within_module_degree=l, among_module_conn=r))

  }

  return(toRol)
}

#' Classify and plot topological roles
#'
#' We use the topological roles calculated with [calc_topological_roles()] and categorize them following the approach of
#' Kortsch (2015), that defines species roles based in two thresholds \eqn{PC=0.625} and \eqn{dz=2.5}.
#' If a species had at least 60% of links within its own module then \eqn{PC<0.625}, and if it also had \eqn{dz\ge 2.5}, thus it was
#' classified as a module hub, labeled **modhub**. If a species had \eqn{PC<0.625} and \eqn{dz<2.5}, then it was called a peripheral
#' or specialist, labeled **modspe**. Species that had \eqn{PC\ge0.625} and \eqn{dz<2.5} were considered module connectors **modcon**.
#' Finally, if a species had \eqn{PC\ge0.625} and \eqn{dz\ge 2.5}, then it was classified as a super-generalist or hub-connector **hubcon**.
#'
#' @references
#'
#'1. Kortsch, S., Primicerio, R., Fossheim, M., Dolgov, A. V & Aschan, M. (2015). Climate change alters the structure of arctic marine food webs due to poleward shifts of boreal generalists. Proc. R. Soc. B Biol. Sci., 282
#'
#' 1. Saravia, L.A., Marina, T.I., De Troch, M. & Momo, F.R. (2018). Ecological Network assembly: how the regional meta web influence local food webs. bioRxiv, 340430, doi: https://doi.org/10.1101/340430
#'
#' @param tRoles Calculated topological roles with the function [calc_topological_roles()]
#' @param g Igraph network object
#' @param spingB Igraph community object
#' @param plt logical whether a plot is generated
#'
#' @return a data.frame with the classified topological roles
#' @export
#'
#' @importFrom RColorBrewer brewer.pal
#' @importFrom dplyr %>% group_by summarise_all inner_join
#'
#' @examples
#' \dontrun{
#'
#' g <- netData[[2]]
#'
#' tp <- calc_topological_roles(g,nsim=10)
#'
#' classify_topological_roles(tp,g,plt=TRUE)
#' }
classify_topological_roles <- function(tRoles,g,spingB=NULL,plt=FALSE){

  if(is.null(spingB))
    spingB <- cluster_spinglass(g,weights=NA)

  spingB.mem<- spingB$membership
  tRoles <- tRoles %>% group_by(node) %>% summarise_all(mean)
  l <- tRoles$within_module_degree
  r <- tRoles$among_module_conn
  # Plot
  if(plt){

    colbar.FG <- brewer.pal(length(spingB$csize),"Dark2")

    old_par <- par(mfrow=c(1,1))
    par(oma=c(2.7,1.3,0.7,1)) # change outer figure margins
    par(mai=c(1,1,0.7,0.7)) # size of figure margins
    yran <- range(l,na.rm = TRUE,finite=TRUE)
    xran <- range(r)
    plot(l~r, type="n", axes=T ,tck=F, lwd=2, ann=F, cex.axis=1.2, xlim=xran, ylim=yran)
    lines(c(0.625,0.625), yran, col="grey")
    lines(xran, c(2.5, 2.5), col="grey")
    points(r, l, col=colbar.FG[spingB.mem], pch=20, cex=2)
    mtext(2, text="Within module degree", line=4,font=2, cex=1.2)
    mtext(1, text="Among module connectivity",line=4, font=2, cex=1.2)
    axis(1, tck=0.02, lwd=1,las=2,lty=1, labels=F, xlim=c(0,1))
    axis(2,tck=0.02, labels=FALSE)
  }
  # Which are the module hubs: many links within its own module.
  #
  modhub <- which(l>2.5)
  modhub <- modhub[which(l>2.5) %in% which(r<=0.625)]
  modlbl <- vertex_attr(g, "name", index=modhub)
  if(is.null(modlbl))
    modlbl <- modhub
  hub_conn <- data.frame()

  if(length(modhub)) {
    if(plt) text(r[modhub],l[modhub],labels = modlbl,cex=0.7,pos=3)
    hub_conn <- data.frame(type="modhub",node=modhub,name=modlbl)
  }

  #points(r[modhub], l[modhub], cex=4, col="blue", pch=20)

  # Which are the hub connectors: high within and between-module connectivity
  #                              and are classified super-generalists
  #
  modhub <- which(l>2.5)
  modhub <- modhub[which(l>2.5) %in% which(r>0.625)]
  modlbl <- vertex_attr(g,"name",index=modhub)
  if(is.null(modlbl))
    modlbl <- modhub
  if(length(modhub)) {
    if(plt) {
      text(r[modhub],l[modhub],labels = modlbl,cex=0.7,pos=3)
      par(old_par)
    }
  }

  #points(r[modhub], l[modhub], cex=4, col="blue", pch=20)
  if(length(modhub)){
    hub_conn <- rbind(hub_conn, data.frame(type="hubcon",node=modhub,name=modlbl))
  }

  # Which are the module specialist: Few links and most of them within its own module
  #
  modhub <- which(l<=2.5)
  modhub <- modhub[which(l<=2.5) %in% which(r<=0.625)]
  modlbl <- vertex_attr(g,"name",index=modhub)
  if(is.null(modlbl))
    modlbl <- modhub

  hub_conn <- rbind(hub_conn, data.frame(type="modspe",node=modhub,name=modlbl))

  # Which are the module connectors: Few links and between modules
  #
  modhub <- which(l<=2.5)
  modhub <- modhub[which(l<=2.5) %in% which(r>0.625)]
  modlbl <- vertex_attr(g,"name", index=modhub)
  if(is.null(modlbl))
    modlbl <- modhub

  hub_conn <- rbind(hub_conn, data.frame(type="modcon",node=modhub,name=modlbl))
  hub_conn <- hub_conn %>% inner_join(tRoles)
  return(hub_conn)
}

#' Calculation of Modularity for a list of igraph objects
#'
#' The function calculates modularity of the list of networks in the nullDist parameter.
#' Modularity is calculated using the [igraph::cluster_spinglass()]
#' Only works for one component networks.
#'
#'
#' @param ig list of igraph objects to calculate modularity
#' @param ncores number of cores used to compute in parallel, if 0 sequential processing is used.
#' @param weights The weights of the edges. Either a numeric vector or NULL or NA. If it is null and the input graph has a ‘weight’ edge attribute
#'                then that will be used. If NULL and no such attribute is present then the edges will have equal weights.
#'                Set this to NA if the graph was a ‘weight’ edge attribute, but you don't want to use it for community detection.

#'
#' @return a data.frame with the field Modularity
#'
#' @export
#'
#' @import igraph
#' @importFrom future.apply future_lapply
#' @importFrom future sequential multiprocess
#'
#'
#' @examples
#' \dontrun{
#' nullg <- generateERbasal(netData[[1]],10)
#' calc_modularity(nullg)
#' }
#'
calc_modularity <- function(ig,ncores=0,weights=NA){

  if(inherits(ig,"igraph")) {
    ig <- list(ig)
  } else if(class(ig[[1]])!="igraph") {
    stop("parameter ig must be an igraph object")
  }

  registerDoFuture()
  if(ncores) {
    cn <- future::availableCores()
    if(ncores>cn)
      ncores <- cn
    future::plan(multiprocess, workers=ncores)
    on.exit(future::plan(sequential))
  } else {
    future::plan(sequential)
  }
  df <-  future_lapply(ig, function(g)
    {
      m<-cluster_spinglass(g,weights=weights)
      modl <- m$modularity
      data.frame(Modularity=modl)
    }, future.seed=TRUE)
  return(bind_rows(df))
}


# via Houstonwp
library(data.table)

q <- c(0.001,
       0.002,
       0.003,
       0.003,
       0.004,
       0.004,
       0.005,
       0.007,
       0.009,
       0.011)

w <- c(0.05,
       0.07,
       0.08,
       0.10,
       0.14,
       0.20,
       0.20,
       0.20,
       0.10,
       0.04)

P <- 100
S <- 25000
r <- 0.02

dt <- as.data.table(cbind(q,w))

npv <- function(cf, r, S, P) {
  cf[, inforce := shift(cumprod(1 - q - w), fill = 1)
  ][, lapses := inforce * w
  ][, deaths := inforce * q
  ][, claims := deaths * S
  ][, premiums := inforce * P
  ][, ncf := premiums - claims
  ][, d := (1/(1+r))^(.I)
  ][, sum(ncf*d)]
}

npv(dt,r,S,P)
#> [1] 50.32483

microbenchmark::microbenchmark(npv(dt,r,S,P))


path <- '/home/maanitou/programming/fsharp/rufc/Samples/schrodinger/'

content_f <- read.table(paste(path, 'output_schrodinger_forward.txt', sep=""))

factor <- 1000000

Xf <- sapply(content_f, function(x) as.numeric(x)/factor)

par(mfrow=c(1,1))

plot(1:128, Xf[1,], type='l',  xlim=c(0,128), ylim=c(min(Xf),max(Xf)))
lines(1:128, Xf[2,], type='l')

for (i in 1:(nrow(Xf)-1)) {
  plot(1:128, Xf[i,], type='l', col='red', xlim=c(0,128), ylim=c(min(Xf),max(Xf)))
  lines(1:128, Xf[i+1,], type='l', col='black')

  #Sys.sleep(0.001)
}


##' Create null model forecasts
##'
##' Uses a truncated normal distribution fitted to past data points under the
##' assumption of no change. Contains uk-specific code on truncation at the time
##' of data availability
##'
##' @param forecasts existing forecast data frame
##' @param data data to create null model
##' @return data frame with null model forecasts
##' @importFrom dplyr arrange mutate group_by ungroup filter
##' @importFrom tidyr complete expand_grid nest unnest
##' @importFrom purrr map
##' @author Sebastian Funk
fit_null_model <- function(forecasts, data) {

  creation_dates <- forecasts %>%
    arrange(creation_date) %>%
    .$creation_date %>%
    unique

  use_data <- data %>%
    mutate(truncation = if_else(value_type == "death_inc_line", 4, 0),
           truncation = if_else(value_type == "hospital_inc" &
                                geography == "Wales", 2, truncation),
           truncation = if_else(value_type == "hospital_inc" &
                                geography == "Scotland", 4, truncation),
           truncation = if_else(value_type == "hospital_inc" &
                                geography == "Northern Ireland", 9, truncation)) %>%
    group_by(geography, value_type, value_desc, truncation) %>%
    tidyr::complete(value_date =
                      seq(min(value_date), max(value_date), by = "day"),
                    fill = list(value = 0)) %>%
  ungroup()

  null_model_forecast_df <-
    function(x, creation_date, horizon = 22, truncation = 0)
  {
    quant <-
      null_model_forecast_quantiles(x$value, horizon = horizon,
                                    truncation = truncation)
    df <-
      bind_rows(replicate(horizon + truncation, quant, simplify = FALSE)) %>%
      mutate(value_date = creation_date - truncation +
               seq(1, horizon + truncation),
             model = "null") %>%
      gather(quantile, value, starts_with("0")) %>%
      mutate(value = round(value),
             quantile = as.numeric(quantile))
  }

  fc <- use_data %>%
    group_by(geography, value_desc, value_type, truncation) %>%
    complete(value_date = seq(min(value_date), max(value_date), by = "day"),
             fill = list(value = 0)) %>%
    ungroup() %>%
    expand_grid(creation_date = creation_dates) %>%
    filter(value_date <= creation_date) %>%
    group_by(creation_date, geography, value_desc, value_type, truncation) %>%
    nest() %>%
    mutate(forecast = map(data, null_model_forecast_df,
                          creation_date = creation_date,
                          truncation = truncation)) %>%
    unnest(forecast) %>%
    ungroup() %>%
    select(model, geography, value_type, value_desc, creation_date, value_date,
           quantile, value)

  return(fc)
}

##' Makes a null model forecast based on a vector of values
##'
##' Null model assumes last value stays
##' @param values vector of values
##' @param horizon forecast horizon
##' @param truncation truncation of data series
##' @importFrom msm dtnorm qtnorm
##' @return quantiles of null model predictive distribution
##' @author Sebastian Funk
null_model_forecast_quantiles <- function(values, horizon, truncation = 0)
{
  if (truncation > 0) {
    values <- head(values, -truncation)
  }

  tnorm_unc_fit <- function(x, mean, true) {
    return(-sum(dtnorm(x = true, mean = mean, sd = x, lower = 0, log = TRUE)))
  }

  quantiles <- seq(0.05, 0.95, by = 0.05)
  ts <- tail(values, min(horizon, length(values)))

  if (length(ts) > 1) {
    ## null model
    null_ts <- rep(last(ts), horizon + truncation)

    interval <- c(0, max(abs(diff(ts))))

    if (diff(interval) > 0 ) {
      tnorm_sigma <-
        optimise(tnorm_unc_fit, interval = interval,
                 mean = head(ts, -1),
                 true = tail(ts, -1))

      quant <- round(qtnorm(p = quantiles, mean = last(ts),
                            sd = tnorm_sigma$minimum, lower = 0))
    } else {
      quant <- rep(last(ts), length(quantiles))
    }
  } else {
    quant <- rep(last(ts), length(quantiles))
  }

  names(quant) <- quantiles

  return(quant)
}


### Auditing
# The Goal: Reduce audit cost (it is currently a 'binding constraint').
# Making auditing between only a few choices.

# #Cleanup
# rm(list=ls())

# #Load everything
# tryCatch(expr=setwd("~/GitHub/Truthcoin/lib"), error=function(e) setwd(choose.dir(caption="Failed to set working directory automatically. Choose 'Truthcoin/lib' folder:")) )
# source(file="consensus/ConsensusMechanism.r")
#
# Use <- function(package) { if(suppressWarnings(!require(package,character.only=TRUE))) install.packages(package,repos="http://cran.case.edu/") ; require(package,character.only=TRUE) }

# Setup
#
# Data:
#
R <- c(.40, .25, .10, .10, .05, .05, .05)

VM <- matrix( c(1, 0, 0, 0, 0,
                1, 0, 0, 0, 0,
                1, 0, 0, 0, 0,
                1, 0, 0, 0, 0,
                1, 0, 0, 0, 0,
                1, 0, 0, 0, 0,
                1, 0, 0, 0, 0), 7,5,byrow=TRUE)

VM2 <- matrix(c(1, 0, 0, 0, 0,
                1, 0, 0, 0, 0,
                0, 0, 0, 0, 0,
                0, 0, 0, 0, 0,
                0, 0, 0, 0, 0,
                0, 0, 0, 0, 0,
                0, 0, 0, 0, 0), 7,5,byrow=TRUE)

VM3 <- matrix(c(1, 0, 0, 0, 0,
                1, 0, 0, 0, 0,
                0, 1, 0, 1, 0,
                0, 0, 0, 0, 1,
                0, 0, 0, 0, 0,
                0, 0, 0, 0, 1,
                0, 0, 0, 0, 1), 7,5,byrow=TRUE)

VM4 <- matrix(c(1, 0, 0, 0, 0,
                1, 0, 0, 0, 0,
                .5, 0, 0, 0, 0,
                .5, 0, 0, 0, 0,
                0, 0, 0, 0, 0,
                0, 0, 0, 0, 0), 6,5,byrow=TRUE)

VM5 <- matrix(c(0.0, 1.0, 0.81, 0.20, 0.03,
                0.0, 1.0, 0.79, 0.20, 0.04, # confused
                0.0, 1.0, 0.80, 0.20, 0.02,
                0.0, 1.0, 0.80, 0.20, 0.03,
                0.0, 1.0, 0.80, 0.98, 0.55, # liars spamming
                0.0, 1.0, 0.80, 0.97, 0.56, # liars spamming
                0.0, 1.0, 0.80, 0.99, 0.57), # liars spamming
              7, 5, byrow=TRUE)

# drawing it out of the binary ballot - cuts everything in half
VM6 <- matrix(c(0.8, 0.0, 0.0, 0.25, 0.045,
                0.6, 0.5, 0.5, 0.20, 0.030,
                0.4, 1.0, 1.0, 0.15, 0.015),
              3, 5, byrow=TRUE)

R6a <- c(.34,.32,.34)

R2 <- c(.40, .20, .20, .17, .01, .01, .01)

# End Setup

GetUniqueBallots <- function(VoteMatrix, Reputation = DemocracyRep(VoteMatrix)) {
  Use('reshape2')
  # Takes a VoteMatrix and Reputation, and returns a minimal set of unique Ballots.

  Rep_VM <- matrix(c(Reputation,VoteMatrix), nrow = nrow(VoteMatrix))

  # Force Unique Names
  colnames(Rep_VM) <- c( "Rep", paste("D",1:(ncol(Rep_VM)-1), sep=".") )

  # Get the unique ballots ("unique"), but ignore non-unique VTC ownership ( " [,-1]" ).
  Map1 <- data.frame( unique(Rep_VM[,-1]) )

  # Relabel (for upcoming merge)
  names(Map1) <- colnames(Rep_VM)[-1] # Again, we're dropping "Rep"

  # Give each Ballot a LETTER label
  Map2 <- cbind( Map1, "BallotGroup"= factor(LETTERS[1:nrow(Map1)]) )

  # Vote Matrix, with labelled Ballots
  Merg <- merge(Rep_VM,Map2)
  MergSub <- Merg[,c("Rep","BallotGroup")]

  # Aggregate Ballots
  Map3 <- dcast(MergSub, formula = BallotGroup~., fun.aggregate = sum, value.var = "Rep") #
  names(Map3) <- c("BallotGroup","BallotRep")

  Merg2 <- merge(Map3,Map2)
  if( !is.null(colnames(VoteMatrix)) ) names(Merg2) <- c( names(Merg2)[1:2], colnames(VoteMatrix) ) # restore original column names (if the given VM had colnames)
  return(Merg2)
}

# GetUniqueBallots(VoteMatrix = VM3, R)
#   BallotGroup BallotRep D.1 D.2 D.3 D.4 D.5
# 1           A      0.65   1   0   0   0   0
# 2           B      0.10   0   1   0   1   0
# 3           C      0.20   0   0   0   0   1
# 4           D      0.05   0   0   0   0   0

# GetUniqueBallots(VoteMatrix = VM4)
#   BallotGroup BallotRep D.1 D.2 D.3 D.4 D.5
# 1           A 0.3333333 1.0   0   0   0   0
# 2           B 0.3333333 0.5   0   0   0   0
# 3           C 0.3333333 0.0   0   0   0   0

# GetUniqueBallots(VoteMatrix = VM5)
#   BallotGroup BallotRep D.1 D.2  D.3  D.4  D.5
# 1           A 0.1428571   0   1 0.81 0.20 0.03
# 2           B 0.1428571   0   1 0.79 0.20 0.04
# 3           C 0.1428571   0   1 0.80 0.20 0.02
# 4           D 0.1428571   0   1 0.80 0.20 0.03
# 5           E 0.1428571   0   1 0.80 0.98 0.55
# 6           F 0.1428571   0   1 0.80 0.97 0.56
# 7           G 0.1428571   0   1 0.80 0.99 0.57

# GetUniqueBallots(VM6,Reputation = R6a)
#   BallotGroup BallotRep D.1 D.2 D.3  D.4   D.5
# 1           A      0.34 0.8 0.0 0.0 0.25 0.045
# 2           B      0.32 0.6 0.5 0.5 0.20 0.030
# 3           C      0.34 0.4 1.0 1.0 0.15 0.015

# We have succeeded in paritioning the VoteMatrix into selections. Auditors need only choose their favorite Ballot.
#
# Can we do better?
# 1. An attacker will likely distort many Decisions at once, to maximize the Attack Revenue.
# 2. Double-Factory will already sort out only the contested Decisions.
# 3. It takes effort for each Auditor to consider each Ballot. Fewer Ballots = Less Effort, Easier Audit.
# 4. Likely, some attacked-Decisions will be extremely-obviously false (a claim that Romney was elected in 2012).
#
# Auditors should be doing MUCH less work than Voters.
# Reducing to an 0 / 1 Ballot Referendum easily allows reuse of the existing SVD consensus system.
#
# Goal: Reduce many unique Ballots to a maximally-representative set of 2 or 3.
# Strategy: calculated 'Moved Reputation' and minimize it.
#
# EasyAuditing
# ==========================

GetTravelMatrix <- function(UniqueBallotDf) {
  # Takes the results computed by 'GetUniqueBallots' and uses them to build a matrix describing how different the Ballots are from each other.

  nBals <- nrow(UniqueBallotDf)

  DistMat <- matrix(0, nBals, nBals,dimnames = list(UniqueBallotDf$BallotGroup, UniqueBallotDf$BallotGroup))

  for(i in 1:nBals) { # for each Ballot-Group A, B, C...
    for(j in 1:nBals) { # for each other Ballot-Group A, B, C...
      if(i!=j) { # Diagonals will always be zero

        RowDifference <- UniqueBallotDf[i,-1:-2] - UniqueBallotDf[j,-1:-2] # How different are these rows?
        Distance <- sum( abs( RowDifference ) ) # Aggregate distance
        DistMat[i,j] <- Distance

        }
      }
    }

  DistMat

  TravelMat <- diag(UniqueBallotDf$BallotRep) %*% DistMat
  row.names(TravelMat) <- colnames(TravelMat)

  return(TravelMat)
}

# GetTravelMatrix( GetUniqueBallots(VoteMatrix = VM3, R) )
#      A    B    C    D
# A 0.00 1.95 1.30 0.65
# B 0.30 0.00 0.30 0.20
# C 0.40 0.60 0.00 0.20
# D 0.05 0.10 0.05 0.00

# GetTravelMatrix( GetUniqueBallots(VoteMatrix = VM4) )
#           A         B         C
# A 0.0000000 0.1666667 0.3333333
# B 0.1666667 0.0000000 0.1666667
# C 0.3333333 0.1666667 0.0000000

### Now, our pooling algorithm

GetChosen <- function(TravMat, ExtractN = 2) {
  # Takes a matrix of travel-distances ("from" row-Ballot "to" column-Ballot) and finds the two nearest destinations (columns)

  Use('combinat')

  Options <- combn(colnames(TravMat), ExtractN)

  # If there is exactly one option, we know will chose it. Also, there will be no columns (as there will be no matrix, just a vector).
  if( is.null(ncol(Options)) ) return(Options)

  BallotPairs <- vector("numeric", ncol(Options))

  # Get Chosen
  for(i in 1:ncol(Options)) {

    ThisDestination <- Options[,i]

    # remove rows where the rowname matched the entries in this Option-column
    # "In a world where these Ballots are staying put (not going from anywhere to anwhere)."

    TravMatTemp <- TravMat[ , colnames(TravMat) %in% ThisDestination ]   # everyone must come to these rows
    TravMatTemp <- TravMatTemp[ !( row.names(TravMatTemp) %in% ThisDestination ) ,] # these rows aren't going anywhere

    if(class(TravMatTemp) =="numeric") {
      # In this case, we've removed all rows except one.
      BallotPairs[i] <- sum(TravMatTemp) # Total distance
      }

    if(class(TravMatTemp) =="matrix") {
      # Normal case
      ColMins <- apply( TravMatTemp, 1, function(x) x[which.min(x)]) # shortest path to a destination
      BallotPairs[i] <- sum(ColMins) # Total distance
      }

    # If TravMatTemp is of a different class, it is probably empty (and we can leave the distances at zero).
    }

  Chosen <- Options[ , which.min(BallotPairs) ]
  return(Chosen)
}

# GetChosen( GetTravelMatrix( GetUniqueBallots(VoteMatrix = VM3, R) ) )
# [1] "A" "C"
#
# GetChosen( GetTravelMatrix( GetUniqueBallots(VoteMatrix = VM4) ) )
# [1] "A" "C"

GetAuditChoices <- function(VoteMatrix, Reputation = DemocracyRep(VoteMatrix), Phi = .80) {
  # Putting it all together.

  # Call our functions.
  Ballots <- GetUniqueBallots(VoteMatrix, Reputation)
  Distance <- GetTravelMatrix(Ballots)

  # Filter small fish / spam Ballots ( because I will be doing combinatorial math next)
  SeriousCandidates <- Distance[, Ballots$BallotRep >= .05 ] # might change ".05" to the Branch's (1-Phi)

  # Grab the most-representative choices, and don't stop grabbing until you have enough
  CumulativeReputation <- 0
  ExtractN <- 2
  while(CumulativeReputation < Phi) {  # ".80" should definitly be Phi
    Chosen <- GetChosen(SeriousCandidates, ExtractN)
    CumulativeReputation <- sum( Ballots[ Ballots$BallotGroup %in% Chosen , "BallotRep" ] )
    ExtractN <- ExtractN + 1
  }

  # For simplicity
  InChosen <- ( row.names(Distance) %in% Chosen )

  # Create two dataframes
  Choices <- Ballots[ InChosen , ]
  NonChoices <- Ballots[ !(InChosen) , ]

  # Get some info to tell people which Ballots contain which.
  NonChDist <- as.matrix( Distance[ !(InChosen) , InChosen ] )

  NonChoices$SurrogateChoice <- apply( NonChDist, 1, function(x) colnames(NonChDist)[which.min(x)] )

  return(list("Choices"=Choices,"NonChoices"=NonChoices))

}

# GetAuditChoices(VM4)
#
# GetAuditChoices(VM3, R)
# GetAuditChoices(VM3, R2)
#
# GetAuditChoices(VM5)
# GetAuditChoices(VM5, R2)
#
# GetAuditChoices(VM6, R6a)
# Here, we've succeeded in drawing the 'real' ballot (B) off, using 68% of the votes. However, B still has a surrogate choice ("C")
# B will deterministically have a unique surrogate choice (not a "Tie") unless every single Decision was scaled and had a real outcome of exactly ".5" (which is nearly impossible).


#' Function to detect year transitions and calculate cumulative age of model results
#'
#' Takes the result of iterative growth modeling and
#' transforms data from Julian Day (0 - 365) to cumulative
#' day of the shell age by detecting where transitions
#' from one year to the next occur and adding full years
#' (365 days) to simulations in later years.
#' @param resultarray Array containing the full results of
#' the optimized growth model
#' @param plotyearmarkers Should the location of identified year
#' transitions be plotted? \code{TRUE/FALSE}
#' @param export_peakid Should the result of peak identification
#' be plotted? \code{TRUE/FALSE}
#' @param path Export path (defaults to tempdir())
#' @return A new version of the Julian Day tab of the resultarray
#' with Julian Day model estimates replaced by estimates of
#' cumulative age of the record in days.
#' @references package dependencies: zoo 1.8.7; scales 1.1.0; graphics
#' function dependencies: peakid
#' @importFrom graphics plot
#' @importFrom stats median
#' @examples
#' testarray <- array(NA, dim = c(40, 36, 9)) # Create empty array
#' # with correct third dimension
#' windowfill <- seq(50, 500, 50) %% 365 # Create dummy simulation data
#' # (ages) to copy through the array
#' for(i in 6:length(testarray[1, , 1])){
#'     testarray[, i, 3] <- c(windowfill, rep(NA, length(testarray[, 1, 3]) -
#'         length(windowfill)))
#'     windowfill <- c(NA, (windowfill + 51) %% 365)
#' }
#' # Add dummy /code{D} column.
#' testarray[, 1, 3] <- seq(1, length(testarray[, 1, 3]), 1)
#' # Add dummy YEARMARKER column
#' testarray[, 3, 3] <- c(0, rep(c(0, 0, 0, 0, 0, 0, 1), 5), 0, 0, 0, 0)
#' # Add dummy d18Oc column
#' testarray[, 2, 3] <- sin((2 * pi * (testarray[, 1, 3] - 8 + 7 / 4)) / 7)
#' testarray2 <- suppressWarnings(cumulative_day(testarray, FALSE, FALSE, tempdir()))
#' # Apply function on array
#' @export
cumulative_day <- function(resultarray, # Align Day of year results from modeling in different windows to a common time axis
    plotyearmarkers = TRUE, # Plot peak fitting?
    export_peakid = TRUE, # Export data on how the boundaries between years were found?
    path = tempdir()
    ){

    dat <- resultarray[, 1:5, 3] # isolate original data

    # Recognition of the boundaries between years.
    # Method one: Apply sinusoidal function to the julian day simulations
    JDdat <- resultarray[, (length(dat[1, ]) + 1):length(resultarray[1, , 1]), 3] # Isolate julian day simulations
    JDdat <- matrix(sin(2 * pi * (JDdat + 365/4) / 365), ncol = ncol(JDdat)) # Convert julian day to sinusoidal value (end and start of year = 1)
    JDends <- data.frame(Depth = dat[, 1],
        Yearmarker = dat[, 3],
        YEsin = scales::rescale(rowSums(JDdat, na.rm = TRUE), c(0, 1)) # Create depth series of positions which most likely represent a year end, rescale to a scale from 0 to 1
    )

    # Method two: Give weight to the first and final days in reconstructions
    weightvector <- c(seq(10, 1, -1), rep(0, 346), seq(1, 10, 1)) # Create vector of weights to be given to days near the start and end of the year
    JDdat <- round(resultarray[, (length(dat[1, ]) + 1):length(resultarray[1, , 1]), 3]) # Isolate julian day simulations
    JDdat <- matrix(weightvector[JDdat + 1], ncol = ncol(JDdat)) # Apply weighting to starts and ends of the year
    JDends$YEweight <- scales::rescale(rowSums(JDdat, na.rm = TRUE), c(0, 1)) # Add normalized results to depth series

    # Method three: Use instances within the window simulations where the end of year is recorded
    JDdat <- resultarray[, (length(dat[1, ]) + 1):length(resultarray[1, , 1]), 3] # Isolate julian day simulations
    JDdat <- rbind(rep(NA, length(JDdat[1, ])), diff(JDdat) < 0) # Matrix of positions in individual windows where end of year (day 365) is recorded
    YEcount <- rowSums(JDdat, na.rm = TRUE) / max(rowSums(JDdat, na.rm = TRUE), na.rm = TRUE) # Aggregate the counted ends of years in simulations
    smoothfactor <- min(diff(which(JDends[, 2] == 1))) # Define smoothing factor for year end counts based on yearmarkers (take thickness of year with least growth to be conservative)
    YEcount <- c(rep(0, floor(smoothfactor / 2)), zoo::rollmean(YEcount, smoothfactor, align = "center"), rep(0, smoothfactor - floor(smoothfactor / 2) - 1)) # Smooth record of year end counts and pad with zeroes
    JDends$YEcount <- scales::rescale(YEcount, c(0, 1)) # Add normalized results to depth series

    # Method four: Use maxima in d18Oc in original data
    yearpos <- c(1, which(JDends[, 2] == 1), length(JDends$Depth)) # Extract positions of yearmarkers and include start and end of record
    yearpos <- unique(yearpos) # Remove duplicates in yearpos due to yearmarkers on beginning and/or end of record
    YE18O <- vector() # Create vector for the position of the maximum d18O value
    for(m in 1:(length(yearpos) - 1)){ # Loop through positions of yearmarkers
        if(m %in% 2:(length(yearpos) - 2)){ # central part of the record
            maxpos <- which(dat[yearpos[m] : (yearpos[m + 1] - 1), 2] == max(dat[yearpos[m] : (yearpos[m + 1] - 1), 2])) + yearpos[m] - 1 # Find the position of the maximum value in the d18O data in that year
            if(length(maxpos) > 1){
                maxpos = round(stats::median(maxpos)) # Prevent multiple values in maxpos (gives errors further in the calculations)
            }
            days <- seq(1, yearpos[m + 1] - yearpos[m], 1) * 365 / (yearpos[m + 1] - yearpos[m]) # Define sequence of "days" values with length = number of datapoints in the year
            sinusoid <- sin(2 * pi * (days - rep((maxpos - yearpos[m]) / (yearpos[m + 1] - yearpos[m]) * 365 - 365 / 4, length(days))) / 365) # Create sinusoid with peak at peak in d18Oc
            sinusoid[which(days > ((maxpos - yearpos[m]) / (yearpos[m + 1] - yearpos[m]) + 0.5) * 365 | days < ((maxpos - yearpos[m]) / (yearpos[m + 1] - yearpos[m]) - 0.5) * 365)] <- -1 # Assign lowest value (-1) to all datapoints more than 1/2 period away from the maximum to prevent false peaks
            YE18O <- append(YE18O, sinusoid) # add sinusoid values to running vector
        }else if(m == 1){ # beginning of the record
            maxpos <- which(dat[yearpos[m + 1] : (yearpos[m + 2] - 1), 2] == max(dat[yearpos[m + 1] : (yearpos[m + 2] - 1), 2])) + yearpos[m + 1] - 1 # Find the position of the maximum value in the d18O data of the next year (the first year that is complete)
            if(length(maxpos) > 1){
                maxpos = round(stats::median(maxpos)) # Prevent multiple values in maxpos (gives errors further in the calculations)
            }
            days <- seq(yearpos[m] - yearpos[m + 1] + 1, yearpos[m + 2] - yearpos[m + 1], 1) * 365 / (yearpos[m + 2] - yearpos[m + 1]) # Define sequence of "days" values
            sinusoid <- sin(2 * pi * (days - rep((maxpos - yearpos[m + 1]) / (yearpos[m + 2] - yearpos[m + 1]) * 365 - 365 / 4, length(days))) / 365) # Create sinusoid with peak at peak in d18Oc
            sinusoid[which(days > ((maxpos - yearpos[m + 1]) / (yearpos[m + 2] - yearpos[m + 1]) + 0.5) * 365 | days < ((maxpos - yearpos[m + 1]) / (yearpos[m + 2] - yearpos[m + 1]) - 0.5) * 365)] <- -1 # Assign lowest value (-1) to all datapoints more than 1/2 period away from the maximum to prevent false peaks
            YE18O <- append(YE18O, sinusoid[1:(yearpos[m + 1] - yearpos[m])]) # Add sinusoid values for first bit of record to the vector
        }else if(m == (length(yearpos) - 1)){ # end of the record
            maxpos <- which(dat[yearpos[m - 1] : (yearpos[m] - 1), 2] == max(dat[yearpos[m - 1] : (yearpos[m] - 1), 2])) + yearpos[m - 1] - 1 # Find the position of the maximum value in the d18O data of the previous year (the last year that is complete)
            if(length(maxpos) > 1){
                maxpos = round(stats::median(maxpos)) # Prevent multiple values in maxpos (gives errors further in the calculations)
            }
            days <- seq(1, yearpos[m + 1] - yearpos[m - 1], 1) * 365 / (yearpos[m] - yearpos[m - 1]) # Define sequence of "days" values
            sinusoid <- sin(2 * pi * (days - rep((maxpos - yearpos[m - 1]) / (yearpos[m] - yearpos[m - 1]) * 365 - 365 / 4, length(days))) / 365) # Create sinusoid with peak at peak in d18Oc
            sinusoid[which(days > ((maxpos - yearpos[m - 1]) / (yearpos[m] - yearpos[m - 1]) + 0.5) * 365 | days < ((maxpos - yearpos[m - 1]) / (yearpos[m] - yearpos[m - 1]) - 0.5) * 365)] <- -1 # Assign lowest value (-1) to all datapoints more than 1/2 period away from the maximum to prevent false peaks
            YE18O <- append(YE18O, sinusoid[(yearpos[m] - yearpos[m - 1]):(yearpos[m + 1] - yearpos[m - 1])]) # Add sinusoid values for last bit of record to the vector
        }
    }
    JDends$YE18O <- scales::rescale(YE18O, c(0, 1)) # Add normalized results to depth series
    JDends$YEcombined <- scales::rescale(rowSums(JDends[, -c(1,2)]), c(0, 1)) # Combine all four methods of peak recognition into one vector

    # Use peak ID algorhythm to find peaks in the combined vector of year end indicators to use as basis for cumulative day counting in the model results
    wmin <- round(min(diff(which(JDends[,2] == 1))) / 4) # Define starting window for peak recognition as one forth the minimum width of a year
    wmax <- round(min(diff(which(JDends[,2] == 1))) / 2) # Define maximum window for peak recognition as half the minimum width of a year
    YM <- length(which(JDends[,2] == 1)) # Extract number of years in record
    X <- list(w = vector(), p = vector()) # List for storing results
    w <- wmin # Start at minimum window size
    repeat{
        peaks <- peakid(JDends$Depth, JDends$YEcombined, w = w, span = 0.05) # Identify peaks based on current threshold
        if(length(peaks$x) == YM){ # Check if the number of years is correct and break loop if it is
            break
        }else if(w < wmax){ # Check if maximum window is reached
            X$w <- append(X$w, w) # Store window size
            X$p <- append(X$p, length(peaks$x)) # Store peak number
            w <- w + 1 # Increment window size
        }else{
            X$w <- append(X$w, w) # Store window size
            X$p <- append(X$p, length(peaks$x)) # Store peak number
            w <- max(X$w[which(abs(X$p - YM) == min(abs(X$p - YM)))]) # If no windows give the exact number of years, take the window that comes closer (prioritize larger windows in case of a tie)
            peaks <- peakid(JDends$Depth, JDends$YEcombined, w = w, span = 0.05) # Recalculate peak positions with the final chosen window size before breaking the loop
            break
        }
    }
    JDends$peakid <- rep(0, length(JDends[,1])) # Add column for peakid results
    JDends$peakid[peaks$i] <- 1 # Mark the location of peaks found in the time series

    if(plotyearmarkers == TRUE){
        dev.new()
        graphics::plot(JDends$Depth,
            JDends$YEcombined,
            type = "l",
            main = "Peak fitting results",
            xlab = "Depth",
            ylab = "Probability of end of year") # Plot aggregate of yearmarkers
        points(peaks$x, rep(1, length(peaks$x)), col = "red") # Plot location of yearmarkers
    }

    if(export_peakid == TRUE){
        write.csv(JDends, file.path(path, "peakid.csv"))
    }

    # Apply calculation of years in the model on the simulation results
    JDdat <- resultarray[, (length(dat[1, ]) + 1):length(resultarray[1, , 1]), 3] # Isolate julian day simulations
    for(col in 1:length(JDdat[1, ])){ # Loop through all simulation windows
        window <- JDdat[, col] # Isolate simulation window
        peakx <- head(which(peaks$x %in% dat[which(!is.na(window)), 1]), 1) # Find position of the first year end in the window column
        if(length(peakx) == 0){
            if(peaks$x[1] < dat[head(which(!is.na(window)), 1), 1]){ # If no global year transitions fall within the window, check if there are global year transitions before the window and find the last year transition before the first value in the window
                peakx <- max(which(peaks$x < dat[head(which(!is.na(window)), 1), 1])) # Find number of years before start of window
                JDpeak <- window[head(which(!is.na(window)), 1)] # Find the day of year of the first simulated value
                if(JDpeak < 182.5){
                    window <- window + peakx * 365 # If first simulated value co-occurs with first half of the year, add all years before the window
                }else{
                    window <- window + (peakx - 1) * 365 # If year first simulated value co-occurs with last half of the year, simulated values are assumed to belong to the previous year
                }
            } # If all year transitions happen after the window, no year number needs to be added to the window values
        }else{
            JDpeak <- JDdat[tail(which(dat[, 1] == peaks$x[peakx] & !is.na(window)), 1), col] # Find JD simulation belonging to the first year transition
            if(JDpeak < 182.5){
                window <- window + peakx * 365 # If year transition co-occurs with first half of the simulated year, all simulated values are assumed to belong to the next year
            }else{
                window <- window + (peakx - 1) * 365 # If year transition co-occurs with last half of the simulated year, all simulated values are assumed to belong to the previous year
            }
        }
        if(length(which(diff(window) < 0)) > 0){ # Check if year transitions occur within the simulation
            for(i in which(diff(window) < 0)){
                if(length(peakx) > 0){ # Check if global year transitions occur within the window
                    if(i < which(dat[, 1] == peaks$x[peakx])){
                        window[1:i] <- window[1:i] - 365 # If the transition happens before the reference point (peakx), subtract a year's worth of days from the values before to prevent adding a year twice.
                    }else{
                        window[(i + 1):length(window)] <- window[(i + 1):length(window)] + 365 # If the transition happens on or after the reference point (peakx), add one year's worth of days to simulations after each transition
                    }
                }else{
                    window[(i + 1):length(window)] <- window[(i + 1):length(window)] + 365 # If no global transitions occur within the window, add one year's worth of days to simulations after each local transition
                }
            }
        }
        JDdat[, col] <- window # Update the julian day data with the new cumulative day simulations
    }

    # Screen for extreme outliers in JD due to very bad simulations based on first and last modeled values
    firstval <- apply(JDdat, 2, min, na.rm = TRUE) # find first values of all modeling windows
    lastval <- apply(JDdat, 2, max, na.rm = TRUE) # find last values of all modeling windows
    badwindow <- rep(FALSE, ncol(JDdat)) # Vector indicating if window simulation is bad
    for(i in 2:(ncol(JDdat) - 1)){
        if(((abs(firstval[i] - firstval[i - 1]) > 365) & (abs(firstval[i + 1] - firstval[i])) > 365) | ((abs(lastval[i] - lastval[i - 1]) > 365) & (abs(lastval[i + 1] - lastval[i])))){ # Find windows with large (> 1 year) jumps in either first or last value
            badwindow[i] <- TRUE # Label these windows as bad
        }
    }
    JDdat[, badwindow] <- rep(NA, nrow(JDdat)) # Remove values from bad windows to prevent messing up the age model

    result <- resultarray[, , 3]
    result[, 6:length(result[1, ])] <- JDdat # Add the updated cumulative julian day results to the resultarray and export
    return(result)
}


# Copyright 2014-2019 Arnaud Poret
# This work is licensed under the BSD 2-Clause License.
# To view a copy of this license, visit https://opensource.org/licenses/BSD-2-Clause

# number of edges
nedge<-8

# vector declarations
# see below for their meaning
edgelab<-vector(mode="character",length=nedge)
r<-vector(mode="numeric",length=nedge)
w<-vector(mode="numeric",length=nedge)

# edge names
# only relevant if edge states are plotted
# plotting edge states is currently not implemented
edgelab[1]<-"EGF__EGFR"
edgelab[2]<-"HRG__EGFR"
edgelab[3]<-"EGFR__Raf"
edgelab[4]<-"AKT__Raf"
edgelab[5]<-"Raf__ERK"
edgelab[6]<-"EGFR__PI3K"
edgelab[7]<-"ERK__PI3K"
edgelab[8]<-"PI3K__AKT"

# edge reactivity
# for each edge and at each iteration of the simulation: the portion of its state which is updated
# here randomly selected between 0.3 and 0.7 as example
r[1]<-runif(1,0.3,0.7)# EGF__EGFR
r[2]<-runif(1,0.3,0.7)# HRG__EGFR
r[3]<-runif(1,0.3,0.7)# EGFR__Raf
r[4]<-runif(1,0.3,0.7)# AKT__Raf
r[5]<-runif(1,0.3,0.7)# Raf__ERK
r[6]<-runif(1,0.3,0.7)# EGFR__PI3K
r[7]<-runif(1,0.3,0.7)# ERK__PI3K
r[8]<-runif(1,0.3,0.7)# PI3K__AKT

# edge weakening
# for each edge and at each iteration of the simulation: the weakening coefficient applied on the updated portion of its state
# here randomly selected between 0.7 and 1 as example
w[1]<-runif(1,0.7,1)# EGF__EGFR
w[2]<-runif(1,0.7,1)# HRG__EGFR
w[3]<-runif(1,0.7,1)# EGFR__Raf
w[4]<-runif(1,0.7,1)# AKT__Raf
w[5]<-runif(1,0.7,1)# Raf__ERK
w[6]<-runif(1,0.7,1)# EGFR__PI3K
w[7]<-runif(1,0.7,1)# ERK__PI3K
w[8]<-runif(1,0.7,1)# PI3K__AKT

# edge updating function
# for all edges and at each iteration of the simulation: the function which calculates their updated states
fedge<-function(node,k,nedge) {
    y<-vector(mode="numeric",length=nedge)
    y[1]<-node[1,k]# EGF__EGFR
    y[2]<-node[2,k]# HRG__EGFR
    y[3]<-node[3,k]# EGFR__Raf
    y[4]<-node[7,k]# AKT__Raf
    y[5]<-node[4,k]# Raf__ERK
    y[6]<-node[3,k]# EGFR__PI3K
    y[7]<-node[5,k]# ERK__PI3K
    y[8]<-node[6,k]# PI3K__AKT
    return(y)
}