#' Create a bias-corrected distribution model
#'
#' This function implements an occupancy-detection model assuming that:
#' \itemize{
#' 		\item{Detection varies by state but not between counties in a state.
#' 		\item{False detections can occur when the species does not occupy a county but does not occur when it does.
#'		\item{Occupancy is a function of the occupancy of neighboring counties.
#' }
#' @param shape SpatialPolygonsDataFrame
#' @param effort Character, name of field in \code{shape} indicating collection effort.
#' @param detect Character, name of field in \code{shape} indicating number of detections of focal species.
#' @param stateProv Character, name of field with state/province.
#' @param county Character, name of field with county.
#' @param niter Positive integer, number of MCMC iterations (including burn-in). The default is 2000, but this is often too low for most cases (i.e., try numbers in the range 10000, 100000, etc.).
#' @param nburnin Positive integer, number of burn-in samples (less than \code{niter}). The default is 1000, but this is often too low for most cases (i.e., try numbers in the range 10000, 100000, etc.).
#' @param nchains Positive integer, number of MCMC chains (default is 4).
#' @param thin Positive integer, number of MCMC samples by which to thin the results (default is 1; i.e., no thinning). To reduce memory requirements, you can use \code{thin} while increasing \code{niter} and/or \code{nburnin}.
#' @param raf Logical, if \code{TRUE} (default), dynamically change the value for niter using the Raftery-Lewis method for assessing number of iterations needed for convergence. The number used will the the number suggested rounded up to the nearest 1000. The total number of iterations (including burn-in) will be this number plus \code{nburnin}.
#' @param minIter Minimum number of iterations to require (not including burn-in).
#' @param maxIter Maximum number of iterations to allow in output. The final number of recommended iterations will be \code{min(maxIter, rl)} where \code{rl} is the number recommend by the RL method, rounded up to the nearest 1000. If \code{rl} is > \code{maxIter} a warning will be issued.
#' @param nsamples Number of samples desired (after burn-in and thinning). Only used if \code{raf} is \code{TRUE}. Used to reset the value of \code{thin} after estimating the number of iterations using the Raftery-Lewis method.
#' @param na.rm Logical, if \code{TRUE} (default), then remove rows in \code{shape} that have \code{NA} in any input field. If this is \code{FALSE} and \code{NA}s occur in any fields then an error may occur.
#' @param verbose Logical, if \code{TRUE} (default) display progress.
#' @param ... Arguments to pass to \code{\link[nimble]{configureMCMC}}, \code{\link[nimble]{runMCMC}}, and \code{rafLewis}.
#' @return A list object with three elements, one with the MCMC chains, a second with the object \code{shape} with model output appended to the data portion of the object, and a third with model information. The new columns in \code{shape} represent:
#' \itemize{
#' 		\item{\code{psi}} Probability of occurrence
#' 		\item{\code{psi95CI}} 95% credibility interval for probability of occurrence
#' 		\item{\code{p}} Probability of detecting the focal species assuming it is present
#' 		\item{\code{p95CI}} 95% credibility interval for probability of detection
#' }
#' @examples
#' @export

trainBayesODM_pMaxVarByState_psiCar <- function(
	shape,
	effort,
	detect,
	stateProv,
	county,
	niter = 2000,
	nburnin = 1000,
	nchains = 4,
	thin = 1,
	raf = TRUE,
	minIter = 2000,
	maxIter = 11000,
	nsamples = 1000,
	na.rm = TRUE,
	verbose = TRUE,
	...
) {

	### prepare data
	################

		# remove missing
		if (na.rm) {
			ok <- complete.cases(shape@data[ , c(effort, detect)])
			shape <- shape[ok, ]
		}

		### CAR setup for counties

		neighs <- spdep::poly2nb(shape, queen=TRUE)
		neighIndex <- which(!sapply(neighs, function(shape) { length(shape == 1) && (shape == 0) }))
		islandIndex <- which(sapply(neighs, function(shape) { length(shape == 1) && (shape == 0) }))

		hasNeighs <- length(neighIndex) > 0
		hasIslands <- length(islandIndex) > 0

		if (hasIslands) {
			shapeNoIslands <- shape[neighIndex, ]
			neighs <- spdep::poly2nb(shapeNoIslands, queen=TRUE)
		}

		numNeighs <- length(neighIndex)
		numIslands <- length(islandIndex)

		numCounties <- nrow(shape)
		numStates <- length(unique(shape@data[ , stateProv]))

		### input
		data <- list()
		constants <- list(
			numStates = numStates
		)

		inits <- list(
			q = 0.01,
			qConstraint = rep(1, numStates),
			p_max = rep(0.6, numStates),
			p_min_star = rep(0.95, numStates)
		)

		# some counties have neighbors
		if (hasNeighs) {

			# remove islands
			neighsList <- spdep::nb2WB(neighs)
			carAdjCounty <- neighsList$adj
			carWeightCounty <- neighsList$weights
			carNumCounty <- neighsList$num

			data <- c(
				data,
				list(
					y_neigh = shape@data[neighIndex, detect],
					N_neigh = shape@data[neighIndex, effort],
					carAdjCounty = carAdjCounty,
					carWeightCounty = carWeightCounty,
					carNumCounty = carNumCounty
				)
			)

			constants <- c(
				constants,
				list(
					numNeighs = numNeighs,
					state_neigh = as.numeric(as.factor(shape@data[neighIndex, stateProv])),

					lengthAdjCounties = length(carAdjCounty)
				)
			)

			inits <- c(
				inits,
				list(
					p_star_neigh = runif(numNeighs, inits$p_min_star * inits$p_max, inits$p_max),

					z_neigh = rep(1, numNeighs),
					psi_neigh = runif(numNeighs, 0.4, 0.6),

					psi_tau = 0.1,
					psi_car = rnorm(numNeighs)
				)
			)

		}

		# some counties are islands
		if (hasIslands) {

			data <- c(
				data,
				list(
					y_island = shape@data[islandIndex, detect],
					N_island = shape@data[islandIndex, effort]
				)
			)

			numStates_island <- length(shape@data[islandIndex, stateProv])

			constants <- c(
				constants,
				list(
					numIslands = numIslands,
					state_island = as.numeric(as.factor(shape@data[islandIndex, stateProv]))
				)
			)

			psi_island <- runif(numIslands, 0.4, 0.6)

			inits <- c(
				inits,
				list(
					p_star_island = runif(numIslands, 0.1, 0.5),
					z_island = rep(1, numIslands),
					psi_island = psi_island,
					logit_psi_island = logit(psi_island),
					psi_islandMean = 0,
					psi_islandTau = 1
				)
			)

		}

		### monitors
		monitors <- names(inits)
		if (hasNeighs) monitors <- c(monitors, 'p_star_neigh')
		if (hasIslands) monitors <- c(monitors, 'p_star_island')

	### model setup
	###############

		code <- if (hasNeighs & hasIslands) {
			.trainBayesODM_pMaxVarByState_psiCar_neighsIslands
		} else if (hasNeighs & !hasIslands) {
			.trainBayesODM_pMaxVarByState_psiCar_neighsOnly
		} else if (!hasNeighs & hasIslands) {
			.trainBayesODM_pMaxVarByState_psiCar_islandsOnly
		}

		### construct and compile model
		###############################

			model <- nimble::nimbleModel(code=code, constants=constants, data=data, inits=inits, check=TRUE)
			flush.console()

			conf <- nimble::configureMCMC(model, monitors = monitors, print = verbose, enableWAIC = TRUE)

			### modify samplers
			node <- 'q'
			conf$removeSamplers(node)
			conf$addSampler(target=node, type='slice')

			if (hasNeighs) {
				node <- 'psi_tau'
				conf$removeSamplers(node)
				conf$addSampler(target=node, type='slice')
			}

			if (hasIslands) {
				node <- 'psi_islandTau'
				conf$removeSamplers(node)
				conf$addSampler(target=node, type='slice')

				# for (i in 1:numIslands) {
					# node <- paste0('logit_psi_island[', i, ']')
					# conf$removeSamplers(node)
					# conf$addSampler(target=node, type='slice')
				# }
			}

			for (i in 1:numStates) {

				node <- paste0('p_max[', i, ']')
				conf$removeSamplers(node)
				conf$addSampler(target=node, type='slice')

				node <- paste0('p_min_star[', i, ']')
				conf$removeSamplers(node)
				conf$addSampler(target=node, type='slice')

			}

			confBuild <- nimble::buildMCMC(conf)
			compiled <- nimble::compileNimble(model, confBuild)

	### variables to ignore when assessing convergence
	##################################################

		exact <- NULL
		# pattern <- c('qConstraint[[]', 'p_star_neigh[[]', 'p_star_island[[]', 'logit_psi_island[[]')
		pattern <- c('qConstraint[[]')

	### use Raftery-Lewis statistic to see how many iterations are needed
	#####################################################################

		if (raf) {

			rl <- rafLewis(
				comp=compiled$confBuild,
				inits = inits,
				niter = niter,
				nburnin = nburnin,
				nsamples = nsamples,
				thin = thin,
				minIter = minIter,
				maxIter = maxIter,
				retry = TRUE,
				exact = exact,
				pattern = pattern,
				propInsufficient = minUnconverged,
				verbose = verbose
			)

			rl$mcmc <- .removeVariablesFromMcmc(rl$mcmc, exact=exact, pattern=pattern)

			rand <- round(1E6 * runif(1))
			tempFile <- paste0('./temp/temp', rand, '.rda')
			save(rl, file=tempFile)
			rm(rl)
			gc()
			load(tempFile)
			file.remove(tempFile)

		}

		if (rl$sufficient) {

			niter <- rl$recIter
			thin <- rl$recThin

			if (coda::niter(rl$mcmc) < nsamples) {
				if (verbose) omnibus::say('Adding iterations to Raftery-Lewis MCMC chain...')
				add <- (niter - burnin - coda::niter(rl$mcmc)) * thin
				rl$mcmc <- addToMcmc(comp = comp, mcmc = rl$mcmc, inits = inits, add = add, thin = thin)
			}

		}

	### add MCMC chains
	###################

		if (nchains > 1 | !raf) {

			startChain <- if (raf) { 2 } else { 1 }
			thisNumChains <- if (raf) { nchains - 1 } else { nchains }

			mcmc <- runMCMC(compiled$confBuild, niter = niter, nburnin = nburnin, thin = thin, nchains = thisNumChains, inits = inits, progressBar = verbose, samplesAsCodaMCMC = TRUE, summary = FALSE, WAIC = FALSE)
			flush.console()

			mcmc <- .removeVariablesFromMcmc(mcmc, exact=exact, pattern=pattern)

		}

		# combine all chains
		nrowRafMcmc <- coda::niter(rl$mcmc)
		nrowNewMcmc <- coda::niter(mcmc)

		if (nrowRafMcmc > nrowNewMcmc) {
			rl$mcmc <- rl$mcmc[1:nrowNewMcmc, ]
			rl$mcmc <- as.mcmc(rl$mcmc)
		}

		mcmc[[1 + coda::nchain(mcmc)]] <- rl$mcmc
		names(mcmc)[coda::nchain(mcmc)] <- paste0('chain', coda::nchain(mcmc))

		save(mcmc, file=tempFile)
		rm(mcmc)
		gc()
		load(tempFile)
		done <- file.remove(tempFile)

	### WAIC and LOO
	################

		# combine chains
		niter <- coda::niter(mcmc) * nchains
		combined <- mcmc[[1]]
		if (nchains > 1) {
			for (chain in 2:nchains) {
				combined <- rbind(combined, mcmc[[chain]])
			}
		}

		# counties with neighborhoods (not islands)
		if (hasNeighs) {

			neigh_ll <- matrix(NA, nrow=niter, ncol=length(data$y_neigh))

			# detection and occupancy
			for (iter in 1:niter) {

				# detection
				z_neigh <- combined[iter, grepl(colnames(combined), pattern='z_neigh[[]'), drop=TRUE]
				p_max <- combined[iter, grepl(colnames(combined), pattern='p_max[[]'), drop=TRUE]
				p_min_star <- combined[iter, grepl(colnames(combined), pattern='p_min_star[[]'), drop=TRUE]

				q <- combined[iter, 'q', drop=TRUE]

				p_max_ll <- dbeta(p_max, 1, 1, log=TRUE)
				p_min_star_ll <- dbeta(p_min_star, 10, 1, log=TRUE)
				q_ll <- dbeta(q, 1, 2, log=TRUE)

				# CAR component
				psi_tau <- combined[iter, 'psi_tau', drop=TRUE]
				psi_tau_ll <- dgamma(psi_tau, 0.001, 0.001, log=TRUE)

				psi_car <- combined[iter, grepl(colnames(combined), pattern='psi_car[[]')]
				psi_car_ll <- dcar_normal(psi_car, adj=data$carAdjCounty[1:constants$lengthAdjCounties], weights=data$carWeightCounty[1:constants$lengthAdjCounties], num=data$carNumCounty[1:constants$numNeighs], tau=psi_tau, c=3, zero_mean=0, log=TRUE)

				p_star_neigh <- combined[iter, grepl(colnames(combined), pattern='p_star_neigh[[]'), drop=TRUE]
				p_neigh <- z_neigh * p_star_neigh + (1 - z_neigh) * q

				neigh_ll[iter, ] <-
					dbinom(data$y_neigh, size=data$N_neigh, prob=p_neigh, log = TRUE) +	# detection
					p_max_ll[constants$state_neigh] +									# state effect
					q_ll +															# false detection
					psi_car_ll														# occupancy ~ CAR

			} # next iteration

		} # has neighbors

		# island counties
		if (hasIslands) {

			island_ll <- matrix(NA, nrow=niter, ncol=length(data$y_island))

			# detection and occupancy
			for (iter in 1:niter) {

				# detection
				z_island <- combined[iter, grepl(colnames(combined), pattern='z_island[[]'), drop=TRUE]
				p_max <- combined[iter, grepl(colnames(combined), pattern='p_max[[]'), drop=TRUE]
				p_min_star <- combined[iter, grepl(colnames(combined), pattern='p_min_star[[]'), drop=TRUE]

				q <- combined[iter, 'q', drop=TRUE]

				p_max_ll <- dbeta(p_max, 1, 1, log=TRUE)
				p_min_star_ll <- dbeta(p_min_star, 10, 1, log=TRUE)
				q_ll <- dbeta(q, 1, 2, log=TRUE)

				p_star_island <- combined[iter, grepl(colnames(combined), pattern='p_star_island[[]'), drop=TRUE]
				p_island <- z_island * p_star_island + (1 - z_island) * q

				psi_islandMean <- combined[iter, 'psi_islandMean', drop=TRUE]
				psi_islandTau <- combined[iter, 'psi_islandTau', drop=TRUE]

				psi_islandMean_ll ~ dnorm(psi_islandMean, 0, sd=1000)
				psi_islandTau_ll ~ dgamma(psi_islandTau, 1, 0.001)

				island_ll[iter, ] <-
					dbinom(data$y_island, size=data$N_island, prob=p_island, log = TRUE) +
					p_max_ll[constants$state_island] +
					p_min_star_ll[constants$state_island] +
					q_ll +															# false detection
					psi_islandMean_ll +												# psi for islands
					psi_islandTau_ll												# psi for islands

			} # next iteration

		} # if has islands

		# LL of neighbors and islands, one row per iteration, one column per county
		ll <- if (hasNeighs & !hasIslands) {
			neigh_ll
		} else if (!hasNeighs & hasIslands) {
			island_ll
		} else {
			cbind(neigh_ll, island_ll)
		}

		waic <- loo::waic(ll)
		loo <- loo::loo(ll, cores=4)

	### process model output
	########################

		mcmc <- .removeVariablesFromMcmc(mcmc, exact=exact, pattern=c('z_neigh[[]', 'z_island[[]'))

		mcmcModel <- .processBayesODM(shape=shape, mcmc=mcmc, effort=effort, detect=detect, neighIndex=neighIndex, islandIndex=islandIndex, pByState=TRUE, stateProv=stateProv)

		meta <- list(
			niter=niter,
			nburnin=nburnin,
			nchains=nchains,
			thin=thin,
			effort=effort,
			detect=detect,
			stateProv=stateProv,
			county=county,
			hasIslands=hasIslands,
			na.rm=na.rm,
			...
		)

		mcmcModel <- c(mcmcModel, list(code=code), meta=list(meta), waic=list(waic), loo=list(loo))
		mcmcModel

}

.trainBayesODM_pMaxVarByState_psiCar_neighsIslands <- nimble::nimbleCode({

	### likelihood for counties with neighborhoods
	for (g in 1:numNeighs) {

		# detection
		y_neigh[g] ~ dbin(p_neigh[g], N_neigh[g])
		p_neigh[g] <- z_neigh[g] * p_star_neigh[g] + (1 - z_neigh[g]) * q
		p_star_neigh[g] ~ dunif(p_min[state_neigh[g]], p_max[state_neigh[g]])

		# occupancy
		z_neigh[g] ~ dbern(psi_neigh[g])
		logit(psi_neigh[g]) <- psi_car[g]

	}

	# county occupancy CAR
	psi_tau ~ dgamma(0.001, 0.001)
	psi_car[1:numNeighs] ~ dcar_normal(adj=carAdjCounty[1:lengthAdjCounties], weights=carWeightCounty[1:lengthAdjCounties], num=carNumCounty[1:numNeighs], tau=psi_tau, c=3, zero_mean=0)

	### likelihood for island counties
	for (h in 1:numIslands) {

		# detection
		y_island[h] ~ dbin(p_island[h], N_island[h])
		p_island[h] <- z_island[h] * p_star_island[h] + (1 - z_island[h]) * q
		p_star_island[h] ~ dunif(p_min[state_island[h]], p_max[state_island[h]])

		# occupancy
		z_island[h] ~ dbern(psi_island[h])
		logit(psi_island[h]) ~ dnorm(psi_islandMean, sd=psi_islandTau)

	}

	psi_islandMean ~ dnorm(0, tau=0.001)
	psi_islandTau ~ dgamma(1, 0.001)

	# false detection
	q ~ dbeta(1, 2)

	# detection
	for (j in 1:numStates) {
		p_max[j] ~ dbeta(1, 1)
		p_min[j] <- p_min_star[j] * p_max[j]
		p_min_star[j] ~ dbeta(10, 1)
		qConstraint[j] ~ dconstraint(p_min[j] > q)
	}

})

.trainBayesODM_pMaxVarByState_psiCar_neighsOnly <- nimble::nimbleCode({

	### likelihood for counties with neighborhoods
	for (g in 1:numNeighs) {

		# detection
		y_neigh[g] ~ dbin(p_neigh[g], N_neigh[g])
		p_neigh[g] <- z_neigh[g] * p_star_neigh[g] + (1 - z_neigh[g]) * q
		p_star_neigh[g] ~ dunif(p_min[state_neigh[g]], p_max[state_neigh[g]])

		# occupancy
		z_neigh[g] ~ dbern(psi_neigh[g])
		logit(psi_neigh[g]) <- psi_car[g]

	}

	# county occupancy CAR
	psi_tau ~ dgamma(0.001, 0.001)
	psi_car[1:numNeighs] ~ dcar_normal(adj=carAdjCounty[1:lengthAdjCounties], weights=carWeightCounty[1:lengthAdjCounties], num=carNumCounty[1:numNeighs], tau=psi_tau, c=3, zero_mean=0)

	# false detection
	q ~ dbeta(1, 2)

	# detection
	for (j in 1:numStates) {
		p_max[j] ~ dbeta(1, 1)
		p_min[j] <- p_min_star[j] * p_max[j]
		p_min_star[j] ~ dbeta(10, 1)
		qConstraint[j] ~ dconstraint(p_min[j] > q)
	}

})


#' Compute absolute distances between intervals.
#'
#' Computes the absolute distance between the midpoint of each `x` interval and
#' the midpoints of each closest `y` interval.
#'
#' @details Absolute distances are scaled by the inter-reference gap for the
#'   chromosome as follows. For `Q` query points and `R` reference
#'   points on a chromosome, scale the distance for each query point `i` to
#'   the closest reference point by the inter-reference gap for each chromosome.
#'   If an `x` interval has no matching `y` chromosome,
#'   `.absdist` is `NA`.
#'
#'   \deqn{d_i(x,y) = min_k(|q_i - r_k|)\frac{R}{Length\ of\ chromosome}}
#'
#'   Both absolute and scaled distances are reported as `.absdist` and
#'   `.absdist_scaled`.
#'
#' @param x [tbl_interval()]
#' @param y [tbl_interval()]
#' @param genome [tbl_genome()]
#'
#' @return
#' [tbl_interval()] with `.absdist` and `.absdist_scaled` columns.
#'
#' @template stats
#'
#' @family interval statistics
#'
#' @seealso
#' \url{http://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1002529}
#'
#' @examples
#' genome <- read_genome(valr_example('hg19.chrom.sizes.gz'))
#'
#' x <- bed_random(genome, seed = 1010486)
#' y <- bed_random(genome, seed = 9203911)
#'
#' bed_absdist(x, y, genome)
#'
#' @export
bed_absdist <- function(x, y, genome) {
  if (!is.tbl_interval(x)) x <- as.tbl_interval(x)
  if (!is.tbl_interval(y)) y <- as.tbl_interval(y)
  if (!is.tbl_genome(genome)) genome <- as.tbl_genome(genome)

  # establish grouping with shared groups (and chrom)
  groups_xy <- shared_groups(x, y)
  groups_xy <- unique(as.character(c("chrom", groups_xy)))
  groups_vars <- rlang::syms(groups_xy)

  # type convert grouping factors to characters if necessary and ungroup
  x <- convert_factors(x, groups_xy)
  y <- convert_factors(y, groups_xy)

  x <- group_by(x, !!! groups_vars)
  y <- group_by(y, !!! groups_vars)

  if (utils::packageVersion("dplyr") < "0.7.99.9000"){
    x_cpp <- update_groups(x)
    y_cpp <- update_groups(y)
    grp_indexes <- shared_group_indexes(x_cpp, y_cpp)
    res <- dist_impl(x_cpp, y_cpp,
                     grp_indexes$x,
                     grp_indexes$y,
                     distcalc = "absdist")
  } else {
    grp_indexes <- shared_group_indexes(x, y)
    res <- dist_impl(x, y,
                     grp_indexes$x, grp_indexes$y,
                     distcalc = "absdist")
  }

  # convert groups_xy to character vector
  if (!is.null(groups_xy)) {
    groups_xy <- as.character(groups_xy)
  }

  # calculate reference sizes
  genome <- filter(genome, genome$chrom %in% res$chrom)
  genome <- inner_join(genome, get_labels(y), by = c("chrom"))

  ref_points <- summarize(y, .ref_points = n())
  genome <- inner_join(genome, ref_points, by = c("chrom", groups_xy))

  genome <- mutate(genome, .ref_gap = .ref_points / size)
  genome <- select(genome, -size, -.ref_points)

  # calculate scaled reference sizes
  res <- full_join(res, genome, by = c("chrom", groups_xy))
  res <- mutate(res, .absdist_scaled = .absdist * .ref_gap)
  res <- select(res, -.ref_gap)

  # report back original x intervals not found
  x_missing <- anti_join(x, res, by = c("chrom", groups_xy))
  x_missing <- ungroup(x_missing)
  x_missing <- mutate(x_missing, .absdist = NA, .absdist_scaled = NA)
  res <- bind_rows(res, x_missing)

  res <- bed_sort(res)

  res
}


#!/usr/local/bin/env Rscript --vanilla
args <- commandArgs(TRUE)
if (is.null(args[1])) {
  stop("No expression matrix provided!")
}

if (is.null(args[2])) {
  stop("No signature matrix provided!")
}

# CIBERSORT R script v1.04 (last updated 10-24-2016)
# Note: Signature matrix construction is not currently available; use java version for full functionality.
# Author: Aaron M. Newman, Stanford University (amnewman@stanford.edu)
# Requirements:
#       R v3.0 or later. (dependencies below might not work properly with earlier versions)
#       install.packages('e1071')
#       install.pacakges('parallel')
#       install.packages('preprocessCore')
#       if preprocessCore is not available in the repositories you have selected, run the following:
#           source("http://bioconductor.org/biocLite.R")
#           biocLite("preprocessCore")
# Windows users using the R GUI may need to Run as Administrator to install or update packages.
# This script uses 3 parallel processes.  Since Windows does not support forking, this script will run
# single-threaded in Windows.
#
# Usage:
#       Navigate to directory containing R script
#
#   In R:
#       source('CIBERSORT.R')
#       results <- CIBERSORT('sig_matrix_file.txt','mixture_file.txt', perm, QN, absolute, abs_method)
#
#       Options:
#       i)   perm = No. permutations; set to >=100 to calculate p-values (default = 0)
#       ii)  QN = Quantile normalization of input mixture (default = TRUE)
#       iii) absolute = Run CIBERSORT in absolute mode (default = FALSE)
#               - note that cell subsets will be scaled by their absolute levels and will not be
#                 represented as fractions (to derive the default output, normalize absolute
#                 levels such that they sum to 1 for each mixture sample)
#               - the sum of all cell subsets in each mixture sample will be added to the ouput
#                 ('Absolute score'). If LM22 is used, this score will capture total immune content.
#       iv)  abs_method = if absolute is set to TRUE, choose method: 'no.sumto1' or 'sig.score'
#               - sig.score = for each mixture sample, define S as the median expression
#                 level of all genes in the signature matrix divided by the median expression
#                 level of all genes in the mixture. Multiple cell subset fractions by S.
#               - no.sumto1 = remove sum to 1 constraint
#
# Input: signature matrix and mixture file, formatted as specified at http://cibersort.stanford.edu/tutorial.php
# Output: matrix object containing all results and tabular data written to disk 'CIBERSORT-Results.txt'
# License: http://cibersort.stanford.edu/CIBERSORT_License.txt

#dependencies
library(e1071)
library(parallel)
library(preprocessCore)

#Core algorithm
CoreAlg <- function(X, y, absolute, abs_method){

  #try different values of nu
  svn_itor <- 3

  res <- function(i){
    if(i==1){nus <- 0.25}
    if(i==2){nus <- 0.5}
    if(i==3){nus <- 0.75}
    model<-svm(X,y,type="nu-regression",kernel="linear",nu=nus,scale=F)
    model
  }

  if(Sys.info()['sysname'] == 'Windows') out <- mclapply(1:svn_itor, res, mc.cores=1) else
    out <- mclapply(1:svn_itor, res, mc.cores=svn_itor)

  nusvm <- rep(0,svn_itor)
  corrv <- rep(0,svn_itor)

  #do cibersort
  t <- 1
  while(t <= svn_itor) {
    weights = t(out[[t]]$coefs) %*% out[[t]]$SV
    weights[which(weights<0)]<-0
    w<-weights/sum(weights)
    u <- sweep(X,MARGIN=2,w,'*')
    k <- apply(u, 1, sum)
    nusvm[t] <- sqrt((mean((k - y)^2)))
    corrv[t] <- cor(k, y)
    t <- t + 1
  }

  #pick best model
  rmses <- nusvm
  mn <- which.min(rmses)
  model <- out[[mn]]

  #get and normalize coefficients
  q <- t(model$coefs) %*% model$SV
  q[which(q<0)]<-0
  if(!absolute || abs_method == 'sig.score') w <- (q/sum(q)) #relative space (returns fractions)
  if(absolute && abs_method == 'no.sumto1') w <- q #absolute space (returns scores)

  mix_rmse <- rmses[mn]
  mix_r <- corrv[mn]

  newList <- list("w" = w, "mix_rmse" = mix_rmse, "mix_r" = mix_r)

}

#do permutations
doPerm <- function(perm, X, Y, absolute, abs_method){
  itor <- 1
  Ylist <- as.list(data.matrix(Y))
  dist <- matrix()

  while(itor <= perm){
    #print(itor)

    #random mixture
    yr <- as.numeric(Ylist[sample(length(Ylist),dim(X)[1])])

    #standardize mixture
    yr <- (yr - mean(yr)) / sd(yr)

    #run CIBERSORT core algorithm
    result <- CoreAlg(X, yr, absolute, abs_method)

    mix_r <- result$mix_r

    #store correlation
    if(itor == 1) {dist <- mix_r}
    else {dist <- rbind(dist, mix_r)}

    itor <- itor + 1
  }
  newList <- list("dist" = dist)
}

#main function
CIBERSORT <- function(sig_matrix, mixture_file, perm=0, QN=FALSE, absolute=FALSE, abs_method='sig.score'){

  if(absolute && abs_method != 'no.sumto1' && abs_method != 'sig.score') stop("abs_method must be set to either 'sig.score' or 'no.sumto1'")

  #read in data
  #X <- read.table(sig_matrix,header=T,sep="\t",row.names=1,check.names=F)
  X <- read.csv(sig_matrix, sep = "\t", row.names = 1,check.names=F)
  #Y <- read.table(mixture_file, header=T, sep="\t",check.names=F)
  Y <- read.csv(mixture_file, sep = "\t",check.names=F)
  #to prevent crashing on duplicated gene symbols, add unique numbers to identical names
  dups <- dim(Y)[1] - length(unique(Y[,1]))
  if(dups > 0) {
    warning(paste(dups," duplicated gene symbol(s) found in mixture file!",sep=""))
    rownames(Y) <- make.names(Y[,1], unique=TRUE)
  }else {rownames(Y) <- Y[,1]}
  Y <- Y[,-1]

  X <- data.matrix(X)
  Y <- data.matrix(Y)

  #order
  X <- X[order(rownames(X)),]
  Y <- Y[order(rownames(Y)),]

  P <- perm #number of permutations

  #anti-log if max < 50 in mixture file
  if (max(Y, na.rm = TRUE) < 50) {
    Y <- 2^Y
    Y[is.na(Y)] <- 0.0
  }

  #quantile normalization of mixture file
  if(QN == TRUE){
    tmpc <- colnames(Y)
    tmpr <- rownames(Y)
    Y <- normalize.quantiles(Y)
    colnames(Y) <- tmpc
    rownames(Y) <- tmpr
  }

  #store original mixtures
  Yorig <- Y
  Ymedian <- max(median(Yorig),1)

  #intersect genes
  Xgns <- row.names(X)
  Ygns <- row.names(Y)
  YintX <- Ygns %in% Xgns
  Y <- Y[YintX,]
  XintY <- Xgns %in% row.names(Y)
  X <- X[XintY,]

  #standardize sig matrix
  X <- (X - mean(X)) / sd(as.vector(X))

  #empirical null distribution of correlation coefficients
  if(P > 0) {nulldist <- sort(doPerm(P, X, Y, absolute, abs_method)$dist)}

  header <- c('Mixture',colnames(X),"P-value","Correlation","RMSE")
  if(absolute) header <- c(header, paste('Absolute score (',abs_method,')',sep=""))

  output <- matrix()
  itor <- 1
  mixtures <- dim(Y)[2]
  pval <- 9999

  #iterate through mixtures
  while(itor <= mixtures){

    y <- Y[,itor]

    #standardize mixture
    y <- (y - mean(y)) / sd(y)

    #run SVR core algorithm
    result <- CoreAlg(X, y, absolute, abs_method)

    #get results
    w <- result$w
    mix_r <- result$mix_r
    mix_rmse <- result$mix_rmse

    if(absolute && abs_method == 'sig.score') {
      w <- w * median(Y[,itor]) / Ymedian
    }

    #calculate p-value
    if(P > 0) {pval <- 1 - (which.min(abs(nulldist - mix_r)) / length(nulldist))}

    #print output
    out <- c(colnames(Y)[itor],w,pval,mix_r,mix_rmse)
    if(absolute) out <- c(out, sum(w))
    if(itor == 1) {output <- out}
    else {output <- rbind(output, out)}

    itor <- itor + 1

  }

  #save results
  #write.table(rbind(header,output), file="CIBERSORT-Results.txt", sep="\t", row.names=F, col.names=F, quote=F)

  #return matrix object containing all results
  obj <- rbind(header,output)
  obj <- obj[,-1]
  obj <- obj[-1,]
  obj <- matrix(as.numeric(unlist(obj)),nrow=nrow(obj))
  rownames(obj) <- colnames(Y)
  if(!absolute){colnames(obj) <- c(colnames(X),"P-value","Correlation","RMSE")}
  else{colnames(obj) <- c(colnames(X),"P-value","Correlation","RMSE",paste('Absolute score (',abs_method,')',sep=""))}
  obj[, 1:(dim(X)[2])]
}

tryCatch(
    expr = {
        cs <- CIBERSORT(args[2], args[1])
        write.table(t(cs), file="deconvoluted.tsv", quote = FALSE, col.names = NA, sep = "\t")
    },
    error = function(e){
      # (Optional)
      # Do this if an error is caught...
      print(e)
      X <- read.csv(args[2], sep = "\t", row.names = 1)
      Y <- read.csv(args[1], sep = "\t")
      cs <- matrix(0, nrow = length(colnames(Y)) - 1, ncol = length(colnames(X)), dimnames = list(colnames(Y)[2:length(colnames(Y))], colnames(X)))
      write.table(t(cs), file="deconvoluted.tsv", quote = FALSE, col.names = NA, sep = "\t")
    }
)


library(dplyr)
library(CHNOSZ)
library(stringr)

### helper functions

# trims away leading and trailing spaces and condenses multiple spaces between words
trimspace <- function(str){
    gsub("(?<=[\\s])\\s*|^\\s+|\\s+$", "", str, perl=TRUE)
}

# isolate a substring by trimming off the portions before and after it
isolate_block <- function(str, begin_str, end_str){
  return(sub(end_str, "", sub(begin_str, "", str)))
}

### main functions

mine_3o <- function(this_file,
                    this_pressure,
                    rxn_table,
                    get_aq_dist=T,
                    get_mass_contribution=T,
                    get_mineral_sat=T,
                    get_redox=T,
                    get_charge_balance=T,
                    get_ion_activity_ratios=T,
                    get_fugacity=T,
                    get_basis_totals=T,
                    get_solid_solutions=T,
                    get_affinity_energy=T,
                    negative_energy_supplies=F,
                    not_limiting=c("H+", "OH-", "H2O"),
                    mass_contribution_other=T,
                    verbose=1){

  # set directory to rxn_3o folder where .3o files are kept
  setwd("rxn_3o")

  # read .3o file as a string
  fileName <- this_file
  extractme <- readChar(fileName, file.info(fileName)$size) # readChar requires filesize, which is obtained from file.info()$size

  # get sample name
  this_name <- trimspace(isolate_block(extractme, begin_str="^.*\\|Sample:\\s+", end_str="\\|\\n\\|.*$"))

  if(verbose > 1){
    writeLines(paste0("Processing EQ3 output for ", this_name))
  }

  # check if file experienced errors. If so, skip processing the file:
  if (grepl("Normal exit", extractme) == FALSE | grepl("\\* Error", extractme)){
    if(verbose > 0){
      writeLines(paste0("\nSample ", this_name, " experienced errors during speciation:"))

      output_error <- str_extract_all(extractme, regex("\\* Error.*?\n(\n|$)", dotall=T))

      output_error <- lapply(output_error, function(x) gsub("\n", " ", x))
      output_error <- lapply(output_error, str_squish)

      for(i in 1:length(output_error)){
        writeLines(output_error[[i]])
      }

    }

    setwd("../")
    return(list())
  }

  sample_3o <- list()

  sample_3o[["filename"]] <- this_file
  sample_3o[["name"]] <- this_name

  ### Begin mining temperature, pressure, water properties

  # mine params
  sample_3o[["temperature"]] <- isolate_block(str=extractme, begin_str="^.*Temperature=\\s+", end_str="\\s+.*$")
  sample_3o[["pressure"]] <- this_pressure #isolate_block(str=extractme, begin_str="^.*Pressure=\\s+", end_str="\\s+.*$")
  sample_3o[["logact_H2O"]] <- isolate_block(str=extractme, begin_str="^.*Log activity of water=\\s+", end_str="\\s+.*$")
  sample_3o[["H2O_density"]] <- isolate_block(str=extractme, begin_str="^.*Solution density =\\s+", end_str="\\s+.*$")
  sample_3o[["H2O_molality"]] <- 55.348/as.numeric(sample_3o[["H2O_density"]])
  sample_3o[["H2O_log_molality"]] <- log10(sample_3o[["H2O_molality"]])

  ### Begin extracting 'Distribution of Aqueous Solute Species' ###
  if (get_aq_dist){
    # string to isolate the aqueous species distribution section:
    front_trim <- "^.*\n\n\n\n                --- Distribution of Aqueous Solute Species ---\n\n    Species                  Molality    Log Molality   Log Gamma  Log Activity\n\n\\s+"

    # isolate species distribution block
    species_block <- isolate_block(str=extractme, begin_str=front_trim, end_str="\n\n.*$")

    # split into substrings, each representing a separate row in the table
    species_block <- strsplit(species_block, "\n")

    #create an empty data frame to store results
    df <- data.frame(species = character(0),
                      molality = character(0),
                      log_molality = character(0),
                      log_gamma = character(0),
                      log_activity = character(0),
                      stringsAsFactors = FALSE)

    # convert into dataframe
    for(this_row in species_block[[1]]){

      # mine row data
      this_row <- trimspace(this_row)
      this_row_data <- strsplit(this_row, " ")[[1]]

      # create a dataframe with results
      this_df <- data.frame(species=this_row_data[1],
                            molality=this_row_data[2],
                            log_molality=this_row_data[3],
                            log_gamma=this_row_data[4],
                            log_activity=this_row_data[5],
                            stringsAsFactors=FALSE)

      # bind distribution of aqueous species to dataframe
      df <- rbind(df, this_df)

    }

    if(!("H2O" %in% df$species)){
      # add a row for water
      df <- rbind(df, data.frame(species="H2O",
                      molality=sample_3o[["H2O_molality"]],
                      log_molality=sample_3o[["H2O_log_molality"]],
                      log_gamma=1,
                      log_activity=sample_3o[["logact_H2O"]],
                      stringsAsFactors=FALSE))
    }

    # set rownames of aqueous species block as species names
    rownames(df) <- df$species
    df$species <- NULL

    # add aqueous block to this sample data
    sample_3o[["aq_distribution"]] <- df

  } # end of 'aqueous distribution' extraction

  if(get_mass_contribution){
    ### begin extracting 'Major Species by Contribution to Aqueous Mass Balances' ###

    # string to isolate the species saturation section:
    front_trim <- "^.*\n\n\n      --- Major Species by Contribution to Aqueous Mass Balances ---\n\n\n"

    # isolate contribution block
    contrib_block <- isolate_block(str=extractme, begin_str=front_trim, end_str="\n\n\n\n.*$")

    # split into substrings, each representing a separate row in the table
    contrib_block <- strsplit(contrib_block, "\n")
    # remove blank linkes
    contrib_block[[1]] <- contrib_block[[1]][contrib_block[[1]] != ""]

    # loop through rows in this block and mine contributions
    mine_vals <- FALSE
    for(this_row in contrib_block[[1]]){
      if(grepl("Accounting for", this_row)){
        # get basis species for this block
        this_basis <- sub(" Species Accounting for 99% or More of Aqueous ", "", this_row)
      } else if (grepl("Per Cent", this_row)){
        # get ready to mine data for this basis species
        mine_vals <- TRUE
        df_basis <- data.frame(species=character(0),
                               molality=character(0),
                               factor=character(0),
                               percent=character(0),
                               stringsAsFactors = FALSE)
      } else if (mine_vals && !grepl(" - - - - - - - - -", this_row)){
        # mine data from this row
        row_data <- trimspace(this_row)
        row_data <- strsplit(row_data, " ")[[1]]
        df_basis <- rbind(df_basis, data.frame(species=row_data[1], factor=row_data[2],
                        molality=row_data[3], percent=row_data[4], stringsAsFactors = FALSE))
      } else if (grepl(" - - - - - - - - -", this_row)){
        # stop mining for this basis species
        mine_vals <- FALSE
        # specify rownames for this contribution block
        rownames(df_basis) <- df_basis$species
        df_basis$species <- NULL
        # add contribution data to list of sample data
        sample_3o[["mass_contribution"]][[this_basis]] <- df_basis
      }
    }
  } # end 'aqueous contribution' extraction

  ### Begin mining mineral saturation section
  if(get_mineral_sat){

    # string to isolate the mineral saturation section:
    front_trim <- "^.*\n\n\n\n           --- Saturation States of Pure Solids ---\n\n       Phase                      Log Q/K    Affinity, kcal\n\n\\s+"

    # isolate mineral block
    mineral_block <- isolate_block(str=extractme, begin_str=front_trim, end_str="\n\n.*$")

    # split into substrings, each representing a separate row in the table
    mineral_block <- strsplit(mineral_block, "\n")

    # create an empty data frame to store results
    df <- data.frame(mineral = character(0),
                     logQoverK = character(0),
                     affinity = character(0),
                     stringsAsFactors = FALSE)

    # convert into dataframe
    for(this_row in mineral_block[[1]]){
      # get row data
      this_row_data <- strsplit(trimspace(this_row), " ")[[1]]

      # create a dataframe with mined data
      this_df <- data.frame(mineral = this_row_data[1],
                            logQoverK = this_row_data[2],
                            affinity = this_row_data[3],
                            stringsAsFactors = FALSE)

      # bind results to dataframe
      df <- rbind(df, this_df)
    }
    rownames(df) <- df$mineral
    df$mineral <- NULL

    # add mineral saturation block to this sample data
    sample_3o[["mineral_sat"]] <- df

    if(get_solid_solutions){
      if(grepl("--- Saturation States of Hypothetical Solid Solutions ---", extractme)){
        # string to isolate the solid solution saturation section:
        front_trim <- "^.*\n\n\n                --- Saturation States of Hypothetical Solid Solutions ---\n\n"

        # isolate solid solution block
        ss_block <- isolate_block(str=extractme, begin_str=front_trim, end_str="\n\n                     --- Fugacities ---.*$") # end_str might be different if fugacity section is not present

        # split into substrings, each representing a separate solid solution
        ss_block <- strsplit(ss_block, "\n\n\n                --- ")[[1]]

        ss_entries <- list()

        for(ss_entry in ss_block){
        ss_entry <- strsplit(ss_entry, " ---\n\n   ")[[1]]
        ss_name <- ss_entry[1]
        ss_name <- sub("\n                --- ", "", ss_name) # needed to clean up the first ss_entry name
        ss_data <- ss_entry[2]
        ss_data <- sub("Ideal solution\n\n    Component                    x           Log x   Log lambda  Log activity\n\n", "", ss_data)[[1]]
        ss_entry <- strsplit(ss_data, "\n\n\n    Mineral                       Log Q/K         Aff, kcal    State\n\n")

        ss_entry <- lapply(ss_entry, FUN=strsplit, "\n")[[1]]
        ss_entry <- lapply(ss_entry, FUN=trimws, "l")
        ss_entry <- lapply(ss_entry, FUN=strsplit, "[ ]{2,}", perl=TRUE)
        names(ss_entry) <- c("ideal solution", "mineral")

        ideal_sol_df <- data.frame(component=character(),
                                   x=numeric(),
                                   `Log x`=numeric(),
                                   `Log lambda`=numeric(),
                                   `Log activity`=numeric(),
                                   stringsAsFactors = FALSE)
        for(row in ss_entry[["ideal solution"]]){

          x <- suppressWarnings(as.numeric(row[2]))
          if(is.na(x)){
            x <- 0
          }

          ideal_sol_df <- rbind(ideal_sol_df, data.frame(component=as.character(row[1]),
                                                         x = x,
                                                         `Log x` = as.numeric(row[3]),
                                                         `Log lambda` = as.numeric(row[4]),
                                                         `Log activity` = as.numeric(row[5])),
                                                         stringsAsFactors=FALSE)
        }
        names(ideal_sol_df) <- c("component", "x", "Log x", "Log lambda", "Log activity") # rename columns because check.names doesn't work when creating the dataframes
        ss_entry[["ideal solution"]] <- ideal_sol_df

        mineral_ss_df <- data.frame(mineral=character(),
                                    `Log Q/K`=numeric(),
                                    `Aff, kcal`=numeric(),
                                    `State`=character(),
                                    stringsAsFactors = FALSE)
        for(row in ss_entry[["mineral"]]){

          if(length(row) == 3){
            row <- c(row, "")
          }
          mineral_ss_df <- rbind(mineral_ss_df, data.frame(mineral=as.character(row[1]),
                                                           `Log Q/K`=as.numeric(row[2]),
                                                           `Aff, kcal`=as.numeric(row[3]),
                                                           `State`=row[4]),
                                                           stringsAsFactors = FALSE)
        }
        names(mineral_ss_df) <- c("mineral", "Log Q/K", "Aff, kcal", "State") # rename columns because check.names doesn't work when creating the dataframes
        ss_entry[["mineral"]] <- mineral_ss_df

        ss_entries[[ss_name]] <- ss_entry
        }
        sample_3o[["solid_solutions"]] <- ss_entries
      }
    }
  } # end 'mineral saturation affinity' extraction

  ### Begin mining redox data
  if(get_redox){
    # string to isolate the redox section:
    front_trim <- "^.*\n\n\n\n                --- Aqueous Redox Reactions ---\n\n   Couple                           Eh, volts      pe-      log fO2   Ah, kcal\n\n\\s+"

    # isolate redox block
    redox_block <- isolate_block(str=extractme, begin_str=front_trim, end_str="\n\n.*$")

    # split into substrings, each representing a separate row in the table
    redox_block <- strsplit(redox_block, "\n")

    #create an empty data frame to store results
    df <- data.frame(couple = character(0),
                     Eh = character(0),
                     pe = character(0),
                     logfO2 = character(0),
                     Ah = character(0),
                     stringsAsFactors = FALSE)

    # convert into dataframe
    for(this_row in redox_block[[1]]){

      # get row data
      this_row_data <- strsplit(trimspace(this_row), " ")[[1]]

      # create a dataframe with results
      this_df <- data.frame(couple = this_row_data[1],
                            Eh = this_row_data[2],
                            pe = this_row_data[3],
                            logfO2 = this_row_data[4],
                            Ah = this_row_data[5],
                            stringsAsFactors = FALSE)

      # bind results to dataframe
      df <- rbind(df, this_df)
    }
    rownames(df) <- df$couple
    df$couple <- NULL

    # add mineral saturation block to this sample data
    sample_3o[["redox"]] <- df
  } # end redox extraction

  ### begin mining charge balance data
  if(get_charge_balance){
    # string to isolate ionic strength:
    front_trim <- "^.*Ionic strength \\(I\\)=\\s+"

    # isolate ionic strength
    IS <- isolate_block(str=extractme, begin_str=front_trim, end_str="\\s+.*$")
    names(IS) <- "ionic strength"

    # string to isolate stoichiometric ionic strength:
    front_trim <- "^.*Stoichiometric ionic strength=\\s+"

    IS_stoich <- isolate_block(str=extractme, begin_str=front_trim, end_str="\\s+.*$")
    names(IS_stoich) <- "stoichiometric ionic strength"

    # string to isolate the electrical balance section:
    front_trim <- "^.*Sigma\\(mz\\) cations=\\s+"

    elec_block <- isolate_block(str=extractme, begin_str=front_trim, end_str="\n\n.*$")

    # split electrical block into strings and numerics
    elec_block <- strsplit(elec_block, "=\\s+|\n\\s+")[[1]]

    elec_block <- c("sigma(mz) cations"=elec_block[1],
                    "sigma(mz) anions"=elec_block[3],
                    "total charge"=elec_block[5],
                    "mean charge"=elec_block[7],
                    "charge imbalance"=elec_block[9])

    # string to isolate charge balance:
    front_trim <- "^.*The electrical imbalance is:\n\n\\s+"

    cbal_bal <- isolate_block(str=extractme, begin_str=front_trim, end_str="\n\n.*$")

    # split electrical block into strings and numerics
    cbal_block <- strsplit(cbal_bal, " per cent|\n\\s+")[[1]]

    cbal_block <- c("charge imbalance % of total charge"=cbal_block[1],
                    "charge imbalance % of mean charge"=cbal_block[3])

    sample_3o[["charge_balance"]] <- c(IS, IS_stoich, elec_block, cbal_block)
  } # end charge balance extraction

  if(get_ion_activity_ratios){
    ion_ratio_block <- isolate_block(extractme, "^.*--- Ion-H\\+ Activity Ratios ---\n\n", "\n\n.*$")
    ion_ratio_block_split <- strsplit(ion_ratio_block, "\n")[[1]]
    ion_ratio_block_split <- strsplit(ion_ratio_block_split, "=")

    if (!identical(ion_ratio_block_split, character(0))){

    ion_ratio_logs <- trimspace(lapply(ion_ratio_block_split, `[[`, 1))
    ion_ratio_values <- suppressWarnings(as.numeric(lapply(ion_ratio_block_split, `[[`, 2)))

    which_to_divide <- grepl("/", ion_ratio_logs) # which of these ratios divide by H+ (instead of multiply?)
    hydrogen_exponents <- unlist(lapply(lapply(strsplit(gsub("^Log \\( a\\(", "", gsub(" \\)$", "", ion_ratio_logs)), "\\)xx "), `[[`, 2), as.numeric))
    ion <- unlist(lapply(strsplit(gsub("^Log \\( a\\(", "", gsub(" \\)$", "", ion_ratio_logs)), "\\) [x|/] a\\("), `[[`, 1))
    ion_times <- lapply(strsplit(gsub("^Log \\( a\\(", "", gsub(" \\)$", "", ion_ratio_logs)), "\\) x a\\(")[!which_to_divide], `[[`, 1)
    ion_divide <- lapply(strsplit(gsub("^Log \\( a\\(", "", gsub(" \\)$", "", ion_ratio_logs)), "\\) / a\\(")[which_to_divide], `[[`, 1)

    names(ion_ratio_values)[!which_to_divide] <- paste0("Log(", ion_times, " x H+**", hydrogen_exponents[!which_to_divide], ")")
    names(ion_ratio_values)[which_to_divide]  <- paste0("Log(", ion_divide, " / H+**", hydrogen_exponents[which_to_divide], ")")

    ion_ratio_values[is.na(ion_ratio_values)] <- "NA"

    df <- data.frame("values"=ion_ratio_values,
               "H_exponent"=hydrogen_exponents,
               "divide"=which_to_divide,
               "ion"=ion)

    df <- transform(transform(df, values = as.character(values)), values=as.numeric(values))
    df <- transform(df, ion = as.character(ion))
    sample_3o[["ion_activity_ratios"]] <- df
    }
  }

  ### begin fugacity mining
  if(get_fugacity){
    fugacity_block <- isolate_block(extractme, "^.*--- Fugacities ---\n\n", "\n\n\n.*$")
    str <- str_squish(strsplit(fugacity_block, "\n")[[1]])
    str <- str[3:length(str)]
    split_str <- strsplit(str, " ")
    df <- data.frame(gas=unlist(lapply(split_str, `[[`, 1)),
                     log_fugacity=as.numeric(unlist(lapply(split_str, `[[`, 2))),
                     stringsAsFactors=F, row.names=1)

    # fix an annoying behavior with R dataframes ignoring rownames when there is
    # only one row
    if(nrow(df) == 1){
      rownames(df) <- unlist(lapply(split_str, `[[`, 1))[1]
      df <- df[ , !(names(df)=="gas")]
    }

    sample_3o[["fugacity"]] <- df
  }

  ### begin sensible composition mining ("basis totals")
  if(get_basis_totals){
    sc_block <- isolate_block(extractme, "^.*--- Sensible Composition of the Aqueous Solution ---\n\n", "\n\n   The above data have.*$")
    str <- str_squish(strsplit(sc_block, "\n")[[1]])
    str <- str[3:length(str)]
    split_str <- strsplit(str, " ")
    sc_names <- unlist(lapply(lapply(split_str, `[[`, 1), paste0, "_total"))
    df <- data.frame(species=sc_names,
                     `mg/L`=as.numeric(unlist(lapply(split_str, `[[`, 2))),
                     `mg/kg.sol`=as.numeric(unlist(lapply(split_str, `[[`, 3))),
                     `molarity`=as.numeric(unlist(lapply(split_str, `[[`, 4))),
                     `molality`=as.numeric(unlist(lapply(split_str, `[[`, 5))),
                     stringsAsFactors=F, row.names=1)

    # fix an annoying behavior with R dataframes ignoring rownames when there is
    # only one row
    if(nrow(df) == 1){
      rownames(df) <- sc_names[1]
      df <- df[ , !(names(df)=="species")]
    }

    sample_3o[["basis_totals"]] <- df

  }

  ### begin energy mining
  if(get_affinity_energy){

    this_temp <- as.numeric(sample_3o[["temperature"]])
    this_logact_H2O <- as.numeric(sample_3o[["logact_H2O"]])
    this_H2O_log_molality <- as.numeric(sample_3o[["H2O_log_molality"]])

    df <- sample_3o[["aq_distribution"]]

    # get a list of mineral names in CHNOSZ
    CHNOSZ_cr_names <- unlist(thermo()$OBIGT %>% filter(state == "cr") %>% select(name))

    # clear any molal values upon moving to this sample
    remaining_react_molal_with_product_molal <- c()
    other_reactants_and_prod_names <- c()
    other_reactants_and_prod <- c()

    # create a dataframe for storing results
    df_rxn <- data.frame(rxn = character(0),
                         affinity = numeric(0),
                         energy_supply = numeric(0),
                         mol_rxn = numeric(0),
                         electrons= numeric(0),
                         reaction = character(0),
                         limiting = character(0),
                         stringsAsFactors = FALSE)

    for(rxn in rxn_table){

      rxn_split <- unlist(strsplit(rxn, "\t"))
      rxn_name  <- rxn_split[1]
      electrons <- as.numeric(rxn_split[2])
      full_rxn  <- rxn_split[3:length(rxn_split)]
      if(length(full_rxn) %% 2 != 0){
        stop(paste("Error: Number of reaction coefficients and species do not match in reaction", rxn_name))
      }

      stoichs <- as.numeric(full_rxn[c(TRUE, FALSE)])
      species_EQ3 <- full_rxn[c(FALSE, TRUE)]
#       species_CHNOSZ <- gsub(",AQ", "", species_EQ3) # this won't work for mineral names, gases, etc.!
#       species_CHNOSZ <- gsub("METHANE", "CH4", species_CHNOSZ) # temporary fix for methane
#       species_CHNOSZ <- gsub("SULFUR", "S", species_CHNOSZ) # temporary fix for sulfur
#       species_CHNOSZ <- gsub("Ca(CO3)", "CaCO3", species_CHNOSZ) # temporary fix for calcium carbonate
#       species_CHNOSZ <- gsub("Ca(CO3),AQ", "CaCO3", species_CHNOSZ) # temporary fix for calcium carbonate

      species_CHNOSZ <- species_EQ3 # assuming no differences between EQ3 and CHNOSZ naming

      # handle minerals
      for(species in species_CHNOSZ){
        if(species %in% CHNOSZ_cr_names){
          #species_CHNOSZ <- gsub(species, lowercase_species, species_CHNOSZ)
          # add the mineral to master_df and master_df_mol dataframes assuming an
          # activity of 1 (log activity 0)
          df[species, "log_activity"] <- 0
          not_limiting <- c(not_limiting, species)
        }
      }

      ### calculate Q using EQ3-speciated activities
      # get speciated activities from master_df
      activities <- c()
      molalities <- c()

      for(species in species_EQ3){

        if(species %in% rownames(df)){

          activities <- c(activities, 10^as.numeric(df[species, "log_activity"]))

          if(!grepl("sub$", rxn_name)){
            molalities <- c(molalities, as.numeric(df[species, "molality"]))
          } else {
            molalities <- c(molalities, remaining_react_molal_with_product_molal[species])
          }

        } else {

          activities <- c(activities, NA)
          molalities <- c(molalities, NA)
        }
      }

      if(!(NA %in% activities) & !is.null(molalities) & !is.null(activities)){

        names(activities) <- species_EQ3
        names(molalities) <- species_EQ3

        if(grepl("sub$", rxn_name)){
          other_reactants_and_prod_names <- setdiff(names(remaining_react_molal_with_product_molal), names(molalities))
          other_reactants_and_prod <- remaining_react_molal_with_product_molal[other_reactants_and_prod_names]
        }

        # calculate Q
        reactant_stoich <- stoichs[which(stoichs < 0)]
        reactant_activities <- activities[which(stoichs < 0)]
        reactant_molalities <- molalities[which(stoichs < 0)]
        product_stoich <- stoichs[which(stoichs > 0)]
        product_activities <- activities[which(stoichs > 0)]
        product_molalities <- molalities[which(stoichs > 0)]
        this_logQ <- sum(abs(product_stoich)*log10(product_activities)) - sum(abs(reactant_stoich)*log10(reactant_activities))

        if(!is.na(this_logQ)){

          ### calculate K using subcrt() function in CHNOSZ
          this_logK <- suppressMessages(subcrt(species=species_CHNOSZ,
                                               coeff=stoichs,
                                               #state=phase,
                                               T=this_temp,
                                               P=this_pressure)$out$logK)
        }else{
          this_logK <- NA
        }

        ### calculate affinity, A
        affinity_per_mol_rxn <- 0.008314*(this_temp+273.15)*2.302585*(this_logK-this_logQ) # in kJ/mol, A=RT*ln(K/Q)=RT*2.302585*(logK-logQ)
        affinity_per_mol_e <- ((affinity_per_mol_rxn*1000)/4.184)/electrons # in cal/mol e-

        ### calculate activity of limiting reactant
        reactant_names <- species_EQ3[which(stoichs < 0)]
        product_names <- species_EQ3[which(stoichs > 0)]
        not_lim_index <- which(reactant_names %in% not_limiting)

        names(reactant_molalities) <- reactant_names

        if(length(not_lim_index) == 0){
          molality_div_stoich <- reactant_molalities/abs(reactant_stoich)
        }else if(all(reactant_names %in% not_limiting)){

          # if there are no limiting reactants (e.g., reactants are all minerals)
          # then append NAs and continue...
          df_rxn <- rbind(df_rxn, data.frame(rxn=rxn_name,
                                             affinity=affinity_per_mol_e,
                                             energy_supply=NA,
                                             mol_rxn=1,
                                             electrons=electrons,
                                             reaction=paste(full_rxn,collapse=" "),
                                             limiting=NA,
                                             stringsAsFactors=FALSE))

          next
        }else{
          molality_div_stoich <- reactant_molalities[-not_lim_index]/abs(reactant_stoich[-not_lim_index])
        }
        limiting_reactant <- min(molality_div_stoich)

        which_limiting <- which(reactant_molalities/abs(reactant_stoich) == limiting_reactant)

        # create a string of all limiting reactants
        limiting_reactants <- paste(reactant_names[which_limiting], collapse=" ")

        ### If there is more than one limiting reactant, pick the first one.
        # Prevents warnings when calculating reactant molalities when the limiting
        # reactant runs out. The math should work out the same regardless of which limiting
        # reactant is chosen.
        which_limiting <- which_limiting[1]

        # calculate moles of rxn before limiting reactant runs out
        mol_rxn <- reactant_molalities[which_limiting] / abs(reactant_stoich[which_limiting])

        # calculate reactant molalities that remain when the limiting reactant runs out
        remaining_reactant_molalities <- reactant_molalities - abs(reactant_stoich) * mol_rxn

        # if 'nonlimiting' species are specified by user, restore their molalities back to their original values.
        if(length(not_lim_index) > 0){
          remaining_reactant_molalities[not_lim_index] <- reactant_molalities[not_lim_index]
        }

        remaining_react_molal_with_product_molal <- c(remaining_reactant_molalities, product_molalities)
        names(remaining_react_molal_with_product_molal) <- c(reactant_names, product_names)

        # attach molalities of other reactants and products that are in the "mother" reaction
        if(grepl("sub$", rxn_name)){
          remaining_react_molal_with_product_molal <- c(remaining_react_molal_with_product_molal, other_reactants_and_prod)
        }

        ### calculate 'energy' in kJ/kg H2O by multiplying affinity (kJ/mol) by activity (mol/kg) of limiting reactant
        this_energy <- affinity_per_mol_rxn * limiting_reactant

        if(!negative_energy_supplies && this_energy < 0){
          this_energy <- 0
        }

      } else { # if there is an NA in one of the activities
        affinity_per_mol_rxn <- NA
        affinity_per_mol_e <- NA
        this_energy <- NA
        mol_rxn <- NA
        limiting_reactants <- NA
      }

      # unit conversion
      energy_supply <- (this_energy*1000)/4.184 # in cal/kg

      # append results
      df_rxn <- rbind(df_rxn, data.frame(rxn=rxn_name,
                                         affinity=affinity_per_mol_e,
                                         energy_supply=energy_supply,
                                         mol_rxn=mol_rxn,
                                         electrons=electrons,
                                         reaction=paste(full_rxn,collapse=" "),
                                         limiting=limiting_reactants,
                                         stringsAsFactors=FALSE))
    } # end rxn loop

    rownames(df_rxn) <- df_rxn$rxn
    df_rxn$rxn <- NULL

    # create a dataframe for storing results
    df_rxn_sum <- data.frame(affinity = numeric(0),
                         energy_supply = numeric(0),
                         electrons = numeric(0),
                         reaction = character(0),
                         stringsAsFactors = FALSE)

    ### sum reaction clusters (a reaction and its subreactions)
    if(sum(grepl("_sub$", rownames(df_rxn))) > 0){
      # perform this chunk of code if "_sub" rxns are present
      rxn_list_sub <- c() # initialize vector of sub-reaction names
      df_rxn[, "mol_rxn_perc"] <- NA # add a new column to df_rxn to store percent mol rxn

      for(rxn in c(rownames(df_rxn), "final_energy")){
        # loop through columns (plus a dummy "final_energy" column)

        if(grepl("_sub", rxn)){
          # if the rxn represents a sub-reaction in the cluster, add column name to vector
          rxn_list_sub <- c(rxn_list_sub, rxn)

        } else {
          # if the rxn does not represent a sub-reaction...
          if(rxn != "final_energy"){
            reaction <- df_rxn[rxn, "reaction"]
          }

          if(length(rxn_list_sub) != 0){
            # sum the previous reaction cluster and append to dataframe
            rxn_sum_sub_E <- colSums(df_rxn[rxn_list_sub, "energy_supply", drop=FALSE])
            rxn_sum_sub_mol <- colSums(df_rxn[rxn_list_sub, "mol_rxn", drop=FALSE])
            rxn_clust_sub_A <- df_rxn[rxn_list_sub, , drop=FALSE] %>% mutate(A_weighted=affinity*(mol_rxn/rxn_sum_sub_mol))
            rxn_clust_sub_mol_perc <- df_rxn[rxn_list_sub, , drop=FALSE] %>% mutate(mol_perc=round(100*(mol_rxn/rxn_sum_sub_mol), 1))
            df_rxn[rxn_list_sub, "mol_rxn_perc"] <- rxn_clust_sub_mol_perc[, "mol_perc"]
            rxn_sum_sub_A <- colSums(rxn_clust_sub_A[, "A_weighted", drop=FALSE])
            rxn_sum_row <- data.frame(row.names=rxn_name, "affinity"=rxn_sum_sub_A, "energy_supply"=rxn_sum_sub_E, electrons=electrons, reaction=reaction)
            df_rxn_sum <- rbind(df_rxn_sum, rxn_sum_row)
          }

          # re-initialize vector for new reaction or reaction cluster
          rxn_name <- rxn
          rxn_list_sub <- c(rxn)

        }
      }
    }else{
      df_rxn_sum <- df_rxn
    }

    # append affinity and energy results to this sample's data
    sample_3o[["affinity_energy_raw"]] <- df_rxn
    sample_3o[["affinity_energy"]] <- df_rxn_sum
  } # end calculation of affinity and energy supply

  setwd("../")

  return(sample_3o)

}

# function to melt aqueous contribution data from multiple samples into
# a single dataframe and then return it.
melt_mass_contribution <- function(batch_3o, other=F, verbose=1){

  # initialize empty dataframe
  df_aq_cont <- data.frame(sample=character(0),
                           basis=character(0),
                           species=character(0),
                           factor=character(0),
                           molality=character(0),
                           percent=character(0),
                           stringsAsFactors=FALSE)

  # get all aqueous contribution data
  mass_contributions <- lapply(batch_3o[["sample_data"]], `[[`, 'mass_contribution')

  # loop through each sample and basis species
  for(sample in names(mass_contributions)){
    if(verbose > 1){
      writeLines(paste0("Processing mass contribution of basis species in ", sample, "..."))
    }
    for(basis in names(mass_contributions[[sample]])){
      df <- mass_contributions[[sample]][[basis]]
      df[, "basis"] <- basis
      df[, "sample"] <- sample
      df[, "species"] <- rownames(df)
      rownames(df) <- NULL

      if(other){
        percent <- round(100-sum(as.numeric(df[, "percent"])), 2)
        df <- rbind(df, data.frame(sample=sample,
                                   basis=basis,
                                   species="Other",
                                   factor=NA,
                                   molality=NA,
                                   percent=toString(percent),
                                   stringsAsFactors=FALSE))
      }

      df_aq_cont <- rbind(df_aq_cont, df)
    }
  }

  df_aq_cont <- df_aq_cont[, c("sample", "basis", "species", "factor", "molality", "percent")]

  return(df_aq_cont)

}

# function to create report versions of data categories (aq distributions, etc.)
create_report_df <- function(data, category, out_type){

  df_cat <- lapply(data, `[[`, category)

  all_species <- unique(unlist(lapply(lapply(df_cat, FUN=t), FUN=colnames)))

  df <- read.csv(text=paste(all_species, collapse="\t"), check.names=FALSE, sep="\t", stringsAsFactors=FALSE)

  for(i in 1:length(df_cat)){
    row <- as.data.frame(t(df_cat[[i]])[out_type, , drop=FALSE], stringsAsFactors=FALSE)
    df <- bind_rows(mutate_all(df, as.character), mutate_all(row, as.character))
  }
  df <- df[, order(colnames(df))]
  df_cat <- cbind.data.frame(sample=names(df_cat), df,  stringsAsFactors = FALSE)

  return(df_cat)

}

# function to compile a report
compile_report <- function(data, csv_filename, aq_dist_type, mineral_sat_type,
                           redox_type, get_aq_dist, get_mineral_sat, get_redox,
                           get_charge_balance, get_ion_activity_ratios, get_fugacity,
                           get_basis_totals, get_affinity_energy, input_processed_df,
                           df_input_processed_names){

  report_list <- list()

  # open processed input file and initialize report with it
  report <- input_processed_df
  names(report) <- df_input_processed_names
  #report <- read.csv(csv_filename, check.names=FALSE, stringsAsFactors=FALSE)

  report_list[["divs"]][["input"]] <- names(report)[2:length(report)] # start at 2 to exclude "sample" column

  # create report versions of EQ3 output blocks
  if(get_aq_dist){
    aq_distribution <- create_report_df(data=data, category='aq_distribution', out_type=aq_dist_type)
    report_list[["divs"]][["aq_distribution"]] <- names(aq_distribution)[2:length(aq_distribution)] # start at 2 to exclude "sample" column
    report <- report %>% inner_join(aq_distribution, by=c("Sample"="sample"))
  }

  if(get_mineral_sat){
    mineral_sat <- create_report_df(data=data, category='mineral_sat', out_type=mineral_sat_type)
    report_list[["divs"]][["mineral_sat"]] <- names(mineral_sat)[2:length(mineral_sat)] # start at 2 to exclude "sample" column
    report <- report %>% inner_join(mineral_sat, by=c("Sample"="sample"))
  }

  if(get_redox){
    redox <- create_report_df(data=data, category='redox', out_type=redox_type)
    report_list[["divs"]][["redox"]] <- names(redox)[2:length(redox)] # start at 2 to exclude "sample" column
    report <- report %>% inner_join(redox, by=c("Sample"="sample"))
  }

  if(get_charge_balance){
    charge_balance <- create_report_df(data=data, category='charge_balance', out_type=1)
    report_list[["divs"]][["charge_balance"]] <- names(charge_balance)[2:length(charge_balance)] # start at 2 to exclude "sample" column
    report <- report %>% inner_join(charge_balance, by=c("Sample"="sample"))
  }

  if(get_ion_activity_ratios){
    if('ion_activity_ratios' %in% names(data)){
    ion_activity_ratios <- create_report_df(data=data, category='ion_activity_ratios', out_type=1)
    report_list[["divs"]][["ion_activity_ratios"]] <- names(ion_activity_ratios)[2:length(ion_activity_ratios)] # start at 2 to exclude "sample" column
    report <- report %>% inner_join(ion_activity_ratios, by=c("Sample"="sample"))
    }
  }

  if(get_fugacity){
    fugacities <- create_report_df(data=data, category='fugacity', out_type=1)
    report_list[["divs"]][["fugacity"]] <- names(fugacities)[2:length(fugacities)] # start at 2 to exclude "sample" column
    report <- report %>% inner_join(fugacities, by=c("Sample"="sample"))
  }

  if(get_basis_totals){
    sc <- create_report_df(data=data, category='basis_totals', out_type=4) # 4 is the molality column
    report_list[["divs"]][["basis_totals"]] <- names(sc)[2:length(sc)] # start at 2 to exclude "sample" column
    report <- report %>% inner_join(sc, by=c("Sample"="sample"))
  }

  if(get_affinity_energy){
    affinity <- create_report_df(data=data, category='affinity_energy', out_type=1)
    names(affinity)[2:length(names(affinity))] <- paste0(names(affinity)[2:length(names(affinity))], "_affinity")
    report_list[["divs"]][["affinity"]] <- names(affinity)[2:length(affinity)] # start at 2 to exclude "sample" column
    energy <- create_report_df(data=data, category='affinity_energy', out_type=2)
    names(energy)[2:length(names(energy))] <- paste0(names(energy)[2:length(names(energy))], "_energy")
    report_list[["divs"]][["energy"]] <- names(energy)[2:length(energy)] # start at 2 to exclude "sample" column
    report <- report %>%
      inner_join(affinity, by=c("Sample"="sample")) %>%
      inner_join(energy, by=c("Sample"="sample"))

  }

  rownames(report) <- report$Sample
  report$Sample <- NULL

  report_list[["report"]] <- report

  return(report_list)

}

### main
main_3o_mine <- function(files_3o,
                         rxn_filename,
                         get_aq_dist,
                         get_mass_contribution,
                         get_mineral_sat,
                         get_redox,
                         get_charge_balance,
                         get_ion_activity_ratios,
                         get_fugacity,
                         get_basis_totals,
                         get_solid_solutions,
                         get_affinity_energy,
                         negative_energy_supplies,
                         load_rxn_file,
                         not_limiting,
                         mass_contribution_other,
                         csv_filename,
                         aq_dist_type,
                         mineral_sat_type,
                         redox_type,
                         input_filename,
                         input_pressures,
                         batch_3o_filename,
                         df_input_processed,
                         df_input_processed_names,
                         custom_obigt,
                         water_model,
                         fixed_species,
                         verbose){

  start_time <- Sys.time()

  water(water_model)

  # allow user to add their custom data as an OBIGT
  if(!is.null(custom_obigt)){
    custom_obigt <- read.csv(custom_obigt, stringsAsFactors=F)
    custom_obigt <- custom_obigt[, c("name", "abbrv", "formula", "state", "ref1", "ref2", "date", "E_units", "G", "H", "S", "Cp", "V", "a1.a", "a2.b", "a3.c", "a4.d", "c1.e", "c2.f", "omega.lambda", "z.T")]
    suppressMessages({
      thermo(OBIGT=thermo()$OBIGT[unique(info(fixed_species)), ]) # replaces the default OBIGT database with user-supplied database
      mod.OBIGT(custom_obigt, replace=TRUE) # produces a message
    })
  }

  rxn_table <- NULL
  if(get_affinity_energy){
    if(load_rxn_file){
      # read table of reactions
      rxn_table <- readLines(rxn_filename)
    }else{
      rxn_table <- strsplit(rxn_filename, "\n")[[1]]
    }
  }

  # instantiate an empty object to store data from all 3o files
  batch_3o <- list()

  if(verbose > 1){
    writeLines("Now processing EQ3 output files...")
  }

  names(input_pressures) <- files_3o

  # process each .3o file
  for(file in files_3o){

    # add this sample's aqueous data to list of all sample data
    sample_3o <- mine_3o(file,
                         this_pressure=input_pressures[file],
                         rxn_table=rxn_table,
                         get_aq_dist=get_aq_dist,
                         get_mass_contribution=get_mass_contribution,
                         get_mineral_sat=get_mineral_sat,
                         get_redox=get_redox,
                         get_charge_balance=get_charge_balance,
                         get_ion_activity_ratios=get_ion_activity_ratios,
                         get_fugacity=get_fugacity,
                         get_basis_totals=get_basis_totals,
                         get_solid_solutions=get_solid_solutions,
                         get_affinity_energy=get_affinity_energy,
                         negative_energy_supplies=negative_energy_supplies,
                         not_limiting=not_limiting,
                         mass_contribution_other=mass_contribution_other,
                         verbose=verbose)

    # if this file could be processed, add its data to the batch_3o object
    if(length(sample_3o)>1){
      batch_3o[["sample_data"]][[sample_3o[["name"]]]] <- sample_3o
    }
  }

  if(verbose > 1){
    writeLines("Finished processing EQ3 output files...")
  }

  # compile aqueous contribution data into a single melted dataframe and
  # append it to the batch_3o object.
  if(get_mass_contribution && length(batch_3o)>0){
    if(verbose > 1){
      writeLines("Now processing mass contribution data...")
    }
    batch_3o[["mass_contribution"]] <- melt_mass_contribution(batch_3o=batch_3o,
                                                              other=mass_contribution_other,
                                                              verbose=verbose)
    if(verbose > 1){
      writeLines("Finished processing mass contribution data...")
    }
  }

  if(length(batch_3o)>0){
    # create a report summarizing 3o data from all samples
    report_list <- compile_report(data=batch_3o[["sample_data"]],
                                  csv_filename=csv_filename,
                                  aq_dist_type,
                                  mineral_sat_type,
                                  redox_type,
                                  get_aq_dist,
                                  get_mineral_sat,
                                  get_redox,
                                  get_charge_balance,
                                  get_ion_activity_ratios,
                                  get_fugacity,
                                  get_basis_totals,
                                  get_affinity_energy,
                                  df_input_processed,
                                  df_input_processed_names)

    # add the report to the batch_3o object
    report <- report_list[["report"]]
    batch_3o[["report"]] <- report
    batch_3o[["report_divs"]] <- report_list[["divs"]]
  }else{
    return(list())
  }

  # store user input file data
  batch_3o[["input"]] <- read.csv(input_filename, check.names=FALSE, stringsAsFactors=FALSE)

  # save the batch_3o object as an rds file
  if(!is.null(batch_3o_filename)){
    saveRDS(batch_3o, file=batch_3o_filename)
  }

  time_elapsed <- Sys.time() - start_time
  if(verbose > 1){
    writeLines(paste("Finished mining .3o files. Time elapsed:", round(time_elapsed, 2), "seconds"))
  }

  return(batch_3o)
}


NAME	HF86

MODE	REDUCTION

SORTS	ANY

SIGNATURE
	a, b, c, d, e, f, g : ANY -> ANY
        sk : -> ANY
ORDERING   KBO
	a = 5, b = 6, c = 6, d = 6, e = 1, f = 1, g = 1, sk = 1
	b > a > c > d > e > f > g > sk

VARIABLES
	x : ANY

EQUATIONS
	 a (e (x)) = e (f (f (f (f (x)))))
	 b (x) = e (f (f (f (x))))
	 c (x) = e (f (f (x)))
	 d (x) = e (f (x))
	 g (e (x)) = x

CONCLUSION
% Loesung :
% e (f (f (f (f (f (f (f (e (f (f (e (f (e (f (g (x)))))))))))))))) =
% e (f (f (f (f (f (f (f (f (f (f (f (f (f (f (e (x))))))))))))))))

         a(b(c(d(e(f(g(sk))))))) = a(c(g(a(a(e(e(sk)))))))


# Install the required R packages
install.packages("vegan")
install.packages("compiler")

library(vegan)    # Contains the function for biodiversity metrics
library(compiler) # Contains the functions to compile the R-commands to byte code to speed up processing

iterations <- 5  # Number of iterations (manuscript used 100 iterations)
ts.length <- 100 # Time series length (manuscript used 100,000 time-steps)
rep.freq <- 10   # Reporting frequency (manuscript reported biodiversity metrics every 1000 time-steps.)

Spec <- 0.0025        # Speciation rate, v (manuscript used these levels: 0.00075, 0.0025, 0.005, 0.01)
Dispersal <- 0.75     # Dispersal rate, m  (manuscript used these levels: 0.25, 0.5, 0.75)
Disp <- (Dispersal/(1-Spec))*1

# Reporters for null counter-factual (sr = species richness; ev = species evenness; beta = community dissimilarity; a = local diversity (alpha), g = regional diversity (gamma))
null.sr.a <- matrix(NA,ncol=(ts.length/rep.freq),nrow=iterations)
null.sr.g <- matrix(NA,ncol=(ts.length/rep.freq),nrow=iterations)
null.ev.a <- matrix(NA,ncol=(ts.length/rep.freq),nrow=iterations)
null.ev.g <- matrix(NA,ncol=(ts.length/rep.freq),nrow=iterations)
null.beta <- matrix(NA,ncol=(ts.length/rep.freq),nrow=iterations)

#Reporters for development (dev) scenario (sr = species richness; ev = species evenness; beta = community dissimilarity; a = local diversity (alpha), g = regional diversity (gamma))
dev.sr.a <- matrix(NA,ncol=(ts.length/rep.freq),nrow=iterations)
dev.sr.g <- matrix(NA,ncol=(ts.length/rep.freq),nrow=iterations)
dev.ev.a <- matrix(NA,ncol=(ts.length/rep.freq),nrow=iterations)
dev.ev.g <- matrix(NA,ncol=(ts.length/rep.freq),nrow=iterations)
dev.beta <- matrix(NA,ncol=(ts.length/rep.freq),nrow=iterations)

# Reporters for restoration (res) scenrio (sr = species richness; ev = species evenness; beta = community dissimilarity; a = local diversity (alpha), g = regional diversity (gamma))
res.sr.a <- matrix(NA,ncol=(ts.length/rep.freq),nrow=iterations)
res.sr.g <- matrix(NA,ncol=(ts.length/rep.freq),nrow=iterations)
res.ev.a <- matrix(NA,ncol=(ts.length/rep.freq),nrow=iterations)
res.ev.g <- matrix(NA,ncol=(ts.length/rep.freq),nrow=iterations)
res.beta <- matrix(NA,ncol=(ts.length/rep.freq),nrow=iterations)

# Reporters for translocation (tran) scenario (sr = species richness; ev = species evenness; beta = community dissimilarity; a = local diversity (alpha), g = regional diversity (gamma))
tran.sr.a <- matrix(NA,ncol=(ts.length/rep.freq),nrow=iterations)
tran.sr.g <- matrix(NA,ncol=(ts.length/rep.freq),nrow=iterations)
tran.ev.a <- matrix(NA,ncol=(ts.length/rep.freq),nrow=iterations)
tran.ev.g <- matrix(NA,ncol=(ts.length/rep.freq),nrow=iterations)
tran.beta <- matrix(NA,ncol=(ts.length/rep.freq),nrow=iterations)

################################################################################
################################################################################

# Function for the forward neutral simulation (death-immigration function)
# Based on: Buschke, F.T. et al. (2016) Ecology and Evolution, doi:10.1002/ece3.2379

death.imm.fun <- function (comm.t,new.K) {
species <- 1:dim(comm.t)[1]
if (runif(1) < Spec) {
  pres.ab <- ifelse(colSums(comm.t)== 0,0,1)
  N.id <- sample(samples,1,prob=(new.K*pres.ab))
  ind.k <- sample(species,1,prob=comm.t[,N.id])
  new.r <- rep(0,N)
  new.r[N.id] <- 1
  comm.t <- rbind(comm.t,new.r)
  if (sum(comm.t[,N.id]) == new.K[N.id]) {
    comm.t[ind.k,N.id] <- comm.t[ind.k,N.id]-1
  }
} else{
  if (runif(1)>Disp) {
      pres.ab <- ifelse(colSums(comm.t)== 0,0,1)
      N.id <- sample(samples,1,prob=(new.K*pres.ab))
      ind.k <- sample(species,1,prob=comm.t[,N.id])
      if (sum(comm.t[,N.id]) == new.K[N.id]) {
        comm.t[ind.k,N.id] <- comm.t[ind.k,N.id]-1
      }
      ind.rep <- sample(species,1,prob=comm.t[,N.id])
      comm.t[ind.rep,N.id] <- comm.t[ind.rep,N.id]+1
  } else {
      pres.ab <- ifelse(colSums(comm.t)== 0,0,1)
      N.id <- sample(samples,1,prob=(new.K*pres.ab))
      ind.k <- sample(species,1,prob=comm.t[,N.id])
      if (sum(comm.t[,N.id]) == new.K[N.id]) {
        comm.t[ind.k,N.id] <- comm.t[ind.k,N.id]-1
      }

      imm.cell <- sample(samples,1,replace=TRUE, prob = (as.vector(imm.prob[N.id,])*pres.ab))

      ind.imm <- sample(species,1,prob=comm.t[,imm.cell])
      comm.t[ind.imm,N.id] <- comm.t[ind.imm,N.id]+1
  }
}
comm.t
}

death.imm <- cmpfun(death.imm.fun)

####################################################################################################
# The follwing code sets up the landscape

for (k in 1:iterations) {
  M <- 179        # Number of patches
  Sites <- 1:M
  K <- rep(100,M) # Habitat capacity of each patch

# Setting up the dispersal kernel based on inverse-quared distance
dist.mat <- matrix(NA,nrow=M,ncol=M)
  for (i in 1:M) {
    for (j in 1:M) {
      if (abs(Sites[i] - Sites[j])<=89) {
        dist.mat[i,j] <- abs(Sites[i] - Sites[j])
      } else {
        dist.mat[i,j] <- 179 - (abs(Sites[i] - Sites[j]))
       }
     }
  }

disp.mat <- 1/(dist.mat^2)
diag(disp.mat) <- 0

###############################################################################

# Create a vector of the site identification for each species unit.
sites <- rep(1:M,K)

J <- sum(K) #  total number of species units

sp.name <- 1:J  # Start by assigning each species unit a unique ID
ind <- 1:J

# This is a probability vector that determines whether a species unit has already coalesed.
ind.prob <- rep(1,J)

# This is simply to set the counter to display the number of time-steps (starts at 1)
x <- 1

################################################################################
# Algorithm for the coalescence process to set up the simulation
# See: Rosindell, J., et al. (2008) Ecological Informatics, 3, 259-271.
################################################################################

# Start the loop and continue until the number of species in the simulation matches the observed species richness
while(sum(ind.prob)> 0) {

# Choose a species unit at random
ind.id <- sample(ind,1,prob=ind.prob)

# First, Speciation
if (runif(1)<Spec) {
  sp.name[which(sp.name==ind.id)] <- sp.name[ind.id]
  # Since the focal species unit already has an assigned species ID, it is excluded from the model from now on.
  ind.prob[ind.id] <- 0
    } else {
  # Second, dispersal
    if (runif(1)>Disp) {
      # Identify the site location of selected species unit
      site.id <- sites[ind.id]
      # Identify a subset of species units that occur in the source patch
      migr.id <- ind[which(sites==site.id)]
        if(length(table(sp.name[migr.id]))>1) {
          # Select a subset of species units that DO NOT already share the same species ID
          imm.id <- migr.id[which(sp.name[migr.id]!=sp.name[ind.id])]
          # Select a parent/immigrant from the source quadrat
          sist.sp <- sample(imm.id,1)
          # Assign the same species ID as the parent species unit to the randomly selected focal species unit
          # AND all other species units that already share that species ID
          sp.name[which(sp.name==ind.id)] <- sp.name[sist.sp]
        }
      # Since the focal bird unit already has an assigned species ID, it is excluded from the model from now on.
      ind.prob[ind.id] <- 0
    } else {
  # Identify the site location of selected species unit
  site.id <- sites[ind.id]
  # Identify the source of selected species unit based on the dispersal kernel
  source.id <- sample(1:M,1,prob=(disp.mat[site.id,]*(K/100)))
  # Identify a subset of species units that occur in the source quadrat
  migr.id <- ind[which(sites==source.id)]
    if(length(table(sp.name[migr.id]))>1) {
      # Select a subset of species units that DO NOT already share the same species ID
      imm.id <- migr.id[which(sp.name[migr.id]!=sp.name[ind.id])]
      # Select a parent/immigrant from the source quadrat
      sist.sp <- sample(imm.id,1)
      # Assign the same species ID as the parent bird unit to the randomly selected focal bird unit
      # AND all other bird units that already share that species ID
       sp.name[which(sp.name==ind.id)] <- sp.name[sist.sp]
     }
    # Since the focal bird unit already has an assigned species ID, it is excluded from the model from now on.
   ind.prob[ind.id] <- 0
   }
  }
}

# Rearrange the new assemblage into a species-by site matrix
Patches <- as.matrix(table(sp.name,sites))

# Remove the diversity at the offset-receiving site (i.e. recent, but unrelated, degraded habitat)
Patches[,81:100] <- 0
K[81:100] <- 0

#####################################################################################
#####################################################################################
# Run the forward siimulation for scenario 1 (counterfactual)

Patches.null <- Patches
imm.prob <- disp.mat

N <- dim(Patches.null)[2]
S <- dim(Patches.null)[1]

species <- seq(1:S)
samples <- seq(1:N)

for (i in 1:ts.length){
  Patches.null <- death.imm(Patches.null,K)
    if (i/rep.freq == floor(i/rep.freq))
      {
      # Summarise all the biodiversity metrics
        null.sr.a[k,i/rep.freq] <- mean(colSums(decostand(Patches.null,"pa")))
        null.sr.g[k,i/rep.freq] <- length(which(rowSums(decostand(Patches.null,"pa"))>0))
        null.ev.a[k,i/rep.freq] <-  mean((diversity(t(Patches.null)))/(log(colSums(decostand(Patches.null,"pa")))),na.rm=TRUE)
        null.ev.g[k,i/rep.freq] <-  mean((diversity(colSums(Patches.null)))/(log(length(which(rowSums(decostand(Patches.null,"pa"))>0)))),na.rm=TRUE)
        null.beta[k,i/rep.freq] <- mean(betadisper(vegdist(t(Patches.null[,which(colSums(Patches.null)>0)])),
        as.factor(rep(1,length(which(colSums(Patches.null)>0)))), type = "centroid")$distances)
      }
  }

#####################################################################################
#####################################################################################
# Run the forward siimulation for scenario 2 (No offsets)

Patches.dev <- Patches
Patches.dev[,c(1:10,170:179)] <- 0
K.dev <- K
K.dev[c(1:10,170:179)] <- 0 # Clear the development site

for (i in 1:ts.length){
  Patches.dev <- death.imm(Patches.dev,K.dev)
    if (i/rep.freq == floor(i/rep.freq))
      {
      # Summarise all the biodiversity metrics
        dev.sr.a[k,i/rep.freq] <- mean(colSums(decostand(Patches.dev,"pa")))
        dev.sr.g[k,i/rep.freq] <- length(which(rowSums(decostand(Patches.dev,"pa"))>0))
        dev.ev.a[k,i/rep.freq] <-  mean((diversity(t(Patches.dev)))/(log(colSums(decostand(Patches.dev,"pa")))),na.rm=TRUE)
        dev.ev.g[k,i/rep.freq] <-  mean((diversity(colSums(Patches.dev)))/(log(length(which(rowSums(decostand(Patches.dev,"pa"))>0)))),na.rm=TRUE)
        dev.beta[k,i/rep.freq] <- mean(betadisper(vegdist(t(Patches.dev[,which(colSums(Patches.dev)>0)])),
        as.factor(rep(1,length(which(colSums(Patches.dev)>0)))), type = "centroid")$distances)
       }
  }

#####################################################################################
#####################################################################################
# Run the forward siimulation for scenario 3 (passive restoration)

Patches.res <- Patches
Patches.res[,c(1:10,170:179)] <- 0
K.res <- K
K.res[c(1:10,170:179)] <- 0   # Clear the development site
K.res[81:100] <- 100          # Increa the habitat capacity at the offset site

for (i in 1:ts.length){
  Patches.res <- death.imm(Patches.res,K.res)
    if (i/rep.freq == floor(i/rep.freq))
      {
      # Summarise all the biodiversity metrics
        res.sr.a[k,i/rep.freq] <- mean(colSums(decostand(Patches.res,"pa")))
        res.sr.g[k,i/rep.freq] <- length(which(rowSums(decostand(Patches.res,"pa"))>0))
        res.ev.a[k,i/rep.freq] <-  mean((diversity(t(Patches.res)))/(log(colSums(decostand(Patches.res,"pa")))),na.rm=TRUE)
        res.ev.g[k,i/rep.freq] <-  mean((diversity(colSums(Patches.res)))/(log(length(which(rowSums(decostand(Patches.res,"pa"))>0)))),na.rm=TRUE)
        res.beta[k,i/rep.freq] <- mean(betadisper(vegdist(t(Patches.res[,which(colSums(Patches.res)>0)])),
        as.factor(rep(1,length(which(colSums(Patches.res)>0)))), type = "centroid")$distances)
      }
   }

#####################################################################################
#####################################################################################
# Run the forward siimulation for scenario 4 (Translocation)

Patches.tran <- Patches
Patches.tran[,81:100] <- Patches.tran[,c(1:10,170:179)] # Translocate the communities
Patches.tran[,c(1:10,170:179)] <- 0  # Clear the development site
K.tran <- K
K.tran[c(1:10,170:179)] <- 0
K.tran[81:100] <- 100

for (i in 1:ts.length){
  Patches.tran <- death.imm(Patches.tran,K.tran)
    if (i/rep.freq == floor(i/rep.freq))
      {
      # Summarise all the biodiversity metrics
        tran.sr.a[k,i/rep.freq] <- mean(colSums(decostand(Patches.tran,"pa")))
        tran.sr.g[k,i/rep.freq] <- length(which(rowSums(decostand(Patches.tran,"pa"))>0))
        tran.ev.a[k,i/rep.freq] <-  mean((diversity(t(Patches.tran)))/(log(colSums(decostand(Patches.tran,"pa")))),na.rm=TRUE)
        tran.ev.g[k,i/rep.freq] <-  mean((diversity(colSums(Patches.tran)))/(log(length(which(rowSums(decostand(Patches.tran,"pa"))>0)))),na.rm=TRUE)
        tran.beta[k,i/rep.freq] <- mean(betadisper(vegdist(t(Patches.tran[,which(colSums(Patches.tran)>0)])),
        as.factor(rep(1,length(which(colSums(Patches.tran)>0)))), type = "centroid")$distances)
      }
  }

  print(k)
flush.console()
}

#################################################################################
# Average the biodiversity metric over the 100 iterations
#################################################################################

m.null.sr.a <- apply(null.sr.a,2,function(x){mean(x,na.rm=TRUE)})
m.null.sr.g <- apply(null.sr.g,2,function(x){mean(x,na.rm=TRUE)})
m.null.ev.a <- apply(null.ev.a,2,function(x){mean(x,na.rm=TRUE)})
m.null.ev.g <- apply(null.ev.g,2,function(x){mean(x,na.rm=TRUE)})
m.null.beta <- apply(null.beta,2,function(x){mean(x,na.rm=TRUE)})

m.dev.sr.a <- apply(dev.sr.a,2,function(x){mean(x,na.rm=TRUE)})
m.dev.sr.g <- apply(dev.sr.g,2,function(x){mean(x,na.rm=TRUE)})
m.dev.ev.a <- apply(dev.ev.a,2,function(x){mean(x,na.rm=TRUE)})
m.dev.ev.g <- apply(dev.ev.g,2,function(x){mean(x,na.rm=TRUE)})
m.dev.beta <- apply(dev.beta,2,function(x){mean(x,na.rm=TRUE)})

m.res.sr.a <- apply(res.sr.a,2,function(x){mean(x,na.rm=TRUE)})
m.res.sr.g <- apply(res.sr.g,2,function(x){mean(x,na.rm=TRUE)})
m.res.ev.a <- apply(res.ev.a,2,function(x){mean(x,na.rm=TRUE)})
m.res.ev.g <- apply(res.ev.g,2,function(x){mean(x,na.rm=TRUE)})
m.res.beta <- apply(res.beta,2,function(x){mean(x,na.rm=TRUE)})

m.tran.sr.a <- apply(tran.sr.a,2,function(x){mean(x,na.rm=TRUE)})
m.tran.sr.g <- apply(tran.sr.g,2,function(x){mean(x,na.rm=TRUE)})
m.tran.ev.a <- apply(tran.ev.a,2,function(x){mean(x,na.rm=TRUE)})
m.tran.ev.g <- apply(tran.ev.g,2,function(x){mean(x,na.rm=TRUE)})
m.tran.beta <- apply(tran.beta,2,function(x){mean(x,na.rm=TRUE)})

#################################################################################
# This plots the outputs from the simulation
# Each plot shows how the log response ratio between the offset scenario and the counterfactual varies through time.

par(mfrow=c(2,3))  # Set up a multi-panel plot with 2 rows and three columns.

# Top-left panel is for local species richness
yval <- seq(rep.freq,ts.length,by=rep.freq)
plot (log(m.dev.sr.a/m.null.sr.a)~yval,type="l",col="red",ylab="Alpha richness",ylim=c(-.2,.2), xlab="Time steps", main="Local species richness")
lines(log(m.res.sr.a/m.null.sr.a)~yval,col="blue")
lines(log(m.tran.sr.a/m.null.sr.a)~yval,col="green")
abline(h=0,lty=2)

# The top-middle panel is for regional species richness
plot (log(m.dev.sr.g/m.null.sr.g)~yval,type="l",col="red",ylab="Gamma richness",ylim=c(-.2,.2), xlab="Time steps", main="Regional species richness")
lines(log(m.res.sr.g/m.null.sr.g)~yval,col="blue")
lines(log(m.tran.sr.g/m.null.sr.g)~yval,col="green")
abline(h=0,lty=2)

# The top-right panel is for local species evenness
plot (log(m.dev.ev.a/m.null.ev.a)~yval,type="l",col="red",ylab="Alpha evenness",ylim=c(-.5,.5), xlab="Time steps", main="Local species evenness")
lines(log(m.res.ev.a/m.null.ev.a)~yval,col="blue")
lines(log(m.tran.ev.g/m.null.ev.g)~yval,col="green")
abline(h=0,lty=2)

# The bottom-left panel is for regional species evenness
plot (log(m.dev.ev.g/m.null.ev.g)~yval,type="l",col="red",ylab="Gamma evennes",ylim=c(-.2,.2), xlab="Time steps", main="Regional species evenness")
lines(log(m.res.ev.g/m.null.ev.g)~yval,col="blue")
lines(log(m.tran.ev.g/m.null.ev.g)~yval,col="green")
abline(h=0,lty=2)

# The bottom-middle panel is for community dissimilarity
plot (log(m.dev.beta/m.null.beta)~yval,type="l",col="red",ylab="Beta dispersion",ylim=c(-.05,.05), xlab="Time steps", main="Community dissimilarity")
lines(log(m.res.beta/m.null.beta)~yval,col="blue")
lines(log(m.tran.beta/m.null.beta)~yval,col="green")
abline(h=0,lty=2)

# This plots a makeshift legen in the bottom right panel
plot(0,0,type="n",xlim=c(0,10),ylim=c(0,10),axes=F, xlab="",ylab="")
lines (c(1,3),c(8,8),lty=2)
text(3.5,8,"No Net loss",pos=4)
lines (c(1,3),c(6,6),col="red")
text(3.5,6,"Scenario 2 (No offset)",pos=4)
lines (c(1,3),c(4,4),col="blue")
text(3.5,4,"Scenario 3 (Passive restoration)",pos=4)
lines (c(1,3),c(2,2),col="green")
text(3.5,2,"Scenario 4 (Translocation)",pos=4)

########################################################################################
########################################################################################

# Write all the outputs to file (will be saved in your default working directory)
write.table(null.sr.a,"Null_alpha.txt",quote=F,row.names=F,sep="\t")
write.table(null.sr.g,"Null_gamma.txt",quote=F,row.names=F,sep="\t")
write.table(null.ev.a,"Null_even_a.txt",quote=F,row.names=F,sep="\t")
write.table(null.ev.g,"Null_even_g.txt",quote=F,row.names=F,sep="\t")
write.table(null.beta,"Null_beta.txt",quote=F,row.names=F,sep="\t")

write.table(dev.sr.a,"Dev_alpha.txt",quote=F,row.names=F,sep="\t")
write.table(dev.sr.g,"Dev_gamma.txt",quote=F,row.names=F,sep="\t")
write.table(dev.ev.a,"Dev_even_a.txt",quote=F,row.names=F,sep="\t")
write.table(dev.ev.g,"Dev_even_g.txt",quote=F,row.names=F,sep="\t")
write.table(dev.beta,"Dev_beta.txt",quote=F,row.names=F,sep="\t")

write.table(res.sr.a,"Res_alpha.txt",quote=F,row.names=F,sep="\t")
write.table(res.sr.g,"Res_gamma.txt",quote=F,row.names=F,sep="\t")
write.table(res.ev.a,"Res_even_a.txt",quote=F,row.names=F,sep="\t")
write.table(res.ev.g,"Res_even_g.txt",quote=F,row.names=F,sep="\t")
write.table(res.beta,"Res_beta.txt",quote=F,row.names=F,sep="\t")

write.table(tran.sr.a,"Tran_alpha.txt",quote=F,row.names=F,sep="\t")
write.table(tran.sr.g,"Tran_gamma.txt",quote=F,row.names=F,sep="\t")
write.table(tran.ev.a,"Tran_even_a.txt",quote=F,row.names=F,sep="\t")
write.table(tran.ev.g,"Tran_even_g.txt",quote=F,row.names=F,sep="\t")
write.table(tran.beta,"Tran_beta.txt",quote=F,row.names=F,sep="\t")


# first make a a determenisti and stochastic forcast and save the results in for.det and for.sto
# load('run/FORECAST.RData'); for.det<-forecast; summary(for.det); summary(for.det[[1]]);
# load('run/FORECAST.RData'); for.sto<-forecast ;  summary(for.sto);  summary(for.sto[[1]]);

# N2013
N2013.sto<-lapply(for.sto,function(x){exp(apply(x$last.state.sim[,1:10],2,median))})
N2013.det<-lapply(for.det,function(x){exp(apply(x$last.state.sim[,1:10],2,median))})
N2013.det
matrix(unlist(N2013.sto),ncol=10,byrow=T)/matrix(unlist(N2013.det),ncol=10,byrow=T)

# N2014
N2014.sto<-lapply(for.sto,function(x){exp(apply(x$s1.state.sim[,1:10],2,median))})
N2014.det<-lapply(for.det,function(x){exp(apply(x$s1.state.sim[,1:10],2,median))})
N2014.det
matrix(unlist(N2014.sto),ncol=10,byrow=T)/matrix(unlist(N2014.det),ncol=10,byrow=T)

SSB2014.sto<-lapply(for.sto,function(x){(apply(x$s1.ssb.sim,2,median))})
SSB2014.det<-lapply(for.det,function(x){(apply(x$s1.ssb.sim,2,median))})
unlist(SSB2014.sto)/unlist(SSB2014.det)

for.det[[1]]$s2.ssb.sim

# N2015
N2015.sto<-lapply(for.sto,function(x){exp(apply(x$s2.state.sim[,1:10],2,median))})
N2015.det<-lapply(for.det,function(x){exp(apply(x$s2.state.sim[,1:10],2,median))})
N2015.det
round(matrix(unlist(N2015.sto),ncol=10,byrow=T)/matrix(unlist(N2015.det),ncol=10,byrow=T),4)

#WB herring
SSB2015.sto<-lapply(for.sto,function(x){median(x$s2.ssb.sim,2)})
SSB2015.det<-lapply(for.det,function(x){median(x$s2.ssb.sim,2)})

SSB2015.sto<-lapply(for.sto,function(x){(apply(x$s2.ssb.sim,2,median))})
SSB2015.det<-lapply(for.det,function(x){(apply(x$s2.ssb.sim,2,median))})

unlist(SSB2015.sto)/unlist(SSB2015.det)

# N2016
N2016.sto<-lapply(for.sto,function(x){exp(apply(x$s3.state.sim[,1:10],2,median))})
N2016.det<-lapply(for.det,function(x){exp(apply(x$s3.state.sim[,1:10],2,median))})
N2016.det
round(matrix(unlist(N2016.sto),ncol=10,byrow=T)/matrix(unlist(N2016.det),ncol=10,byrow=T),4)

SSB2016.sto<-lapply(for.sto,function(x){(apply(x$s3.ssb.sim,2,median))})
SSB2016.det<-lapply(for.det,function(x){(apply(x$s3.ssb.sim,2,median))})

#WB herring
SSB2016.sto<-lapply(for.sto,function(x){median(x$s3.ssb.sim,2)})
SSB2016.det<-lapply(for.det,function(x){median(x$s3.ssb.sim,2)})

round(unlist(SSB2016.sto)/unlist(SSB2016.det),4)

for.det[[1]]$s2.catch.sim
C2015.sto<-lapply(for.sto,function(x){(median(x$s2.catch.sim))})
C2015.det<-lapply(for.det,function(x){(median(x$s2.catch.sim))})
round(unlist(C2015.sto)/unlist(C2015.det),4)

#WB herring
SSB2015.sto<-lapply(for.sto,function(x){median(x$s2.ssb.sim,2)})
SSB2015.det<-lapply(for.det,function(x){median(x$s2.ssb.sim,2)})

# extract N 2015 from the first forecast option
matrix(unlist(N2015.sto),ncol=10,byrow=T)
N<-matrix(unlist(N2015.sto),ncol=10,byrow=T)[1,]
N

# just checing they are the same
for.sto[[1]]$ave.sw
for.det[[1]]$ave.sw

sum(for.sto[[1]]$ave.sw*for.sto[[1]]$ave.pm*N)   # the simple way
N %*% (for.sto[[1]]$ave.sw*for.sto[[1]]$ave.pm)  # the Anders way
SSB2015.sto[[1]]
SSB2015.sto[[1]] / sum(for.sto[[1]]$ave.sw*for.sto[[1]]$ave.pm*N)

sum(for.det[[1]]$ave.sw*for.sto[[1]]$ave.pm*N)   # the simple way
N %*% (for.sto[[1]]$ave.sw*for.sto[[1]]$ave.pm)  # the Anders way
SSB2015.det[[1]]
SSB2015.det[[1]] /sum(for.det[[1]]$ave.sw*for.sto[[1]]$ave.pm*N)

#The right way
lapply(for.det,function(x){apply(exp(x$s2.state.sim[,1:10])*rep(x$ave.sw*x$ave.pm,each=2),1,sum)})
SSB2015.det<-lapply(for.det,function(x){median(apply(exp(x$s2.state.sim[,1:10])*rep(x$ave.sw*x$ave.pm,each=2),1,sum))})

lapply(for.sto,function(x){apply(exp(x$s2.state.sim[,1:10])*rep(x$ave.sw*x$ave.pm,each=1000),1,sum)})
SSB2015.sto<-lapply(for.sto,function(x){median(apply(exp(x$s2.state.sim[,1:10])*rep(x$ave.sw*x$ave.pm,each=1000),1,sum))})

unlist(SSB2015.sto)/unlist(SSB2015.det)

# ###########################################
 library(MASS)

noSim<-30000
var1<-0.8
var2<-var1*0.75
var12<-0.1    # co-var
mu1<-4
mu2<-mu1/2
weight<-c(1,10)
sigma<-matrix(c(var1,var12,var12,var2),ncol=2)
logN<-mvrnorm(noSim, mu=c(mu1,mu2), Sigma=sigma)

plot(logN[,1],logN[,2])
hist(logN[,1])

N<-exp(logN)
N.median<-apply(N,2,median)
#N.median
if (noSim<=10) log(N.median)
if (noSim<=10) c(mu1,mu2)    # there is no bias in N
bio.deter<-exp(c(mu1,mu2))*weight
if (noSim<=10) bio.deter
bio.deter<-sum(bio.deter)
if (noSim<=10) bio.deter

if (noSim<=10) N
bio<-N*rep(weight,each=noSim)
if (noSim<=10) bio

bio<-apply(bio,1,sum)
if (noSim<=10) bio
bio.sto<-median(bio)

bio.sto/bio.deter
#######

# The same, but mu and variance and weight is the same for both of them

noSim<-1000000
sd1<-0.8
var1<-sd1**2
var2<-var1
var12<-var1/4    # co-var
mu1<-4
mu2<-mu1*1
weight<-c(1,1)
sigma<-matrix(c(var1,var12,var12,var2),ncol=2)
logN<-mvrnorm(noSim, mu=c(mu1,mu2), Sigma=sigma)

par(mfcol=c(2,2))
hist(logN[,1],xlab='logN1',main=paste("age 1, logN, mean:",round(mean(logN[,1]),4)))
abline(v=mean(logN[,1]),col='red')

hist(exp(logN[,1]),xlab='exp(logN1)',main=paste("age 1, exp(logN),median:",round(median(exp(logN[,1])),4),'\n exp(logN):',round(exp(mu1),4)))
abline(v=median(exp(logN[,1])),col='red')

hist(logN[,2],xlab='logN2',main=paste("age 2, logN, mean:",round(mean(logN[,2]),4)))
abline(v=mean(logN[,2]),col='red')

hist(exp(logN[,2]),xlab='exp(logN1)',main=paste("age 2, exp(logN),median:",round(median(exp(logN[,2])),4),'\n exp(logN):',round(exp(mu1),4)))
abline(v=median(exp(logN[,2])),col='red')

par(mfcol=c(1,1))

N<-exp(logN)
bio.deter<-exp(c(mu1,mu2))*weight
bio.deter<-sum(bio.deter)
bio.deter

bio<-N*rep(weight,each=noSim)
bio<-apply(bio,1,sum)
bio.sto<-median(bio)
hist(bio,main=paste('median bio:',round(bio.sto,4),'\n determ bio:',round(bio.deter,4),'\n ratio stock:determ',round(bio.sto/bio.deter,4)))
abline(v=bio.sto,col='red')

#######

neafc<-function(sd1=0.8,var2Fac=1,mu1=4,mu2Fac=1,noSim=100000,weight=c(1,10)){
  var1<-sd1*sd1
  var2<-var1*var2Fac
  var12<-var1/5    # co-var

  mu2<-mu1*mu2Fac
  sigma<-matrix(c(var1,var12,var12,var2),ncol=2)
  logN<-mvrnorm(noSim, mu=c(mu1,mu2), Sigma=sigma)
  N<-exp(logN)
  bio.sto<-N*rep(weight,each=noSim)
  bio.sto.corrected<-bio.sto # "corrected" later on

  bio.sto<-apply(bio.sto,1,sum)
  bio.sto<-median(bio.sto)

  #cat('\nbefore adjustement')
  #print(bio.sto.corrected)

  cor.fac<-exp(rep(0.5*c(var1,var2),each=noSim))
  #print(cor.fac)

  bio.sto.corrected<-bio.sto.corrected/cor.fac

  #cat('\nafter adjustement\n')
  #print(bio.sto.corrected)
   bio.sto.corrected<-apply(bio.sto.corrected,1,sum)
  #cat('\nsum after adjustement\n')
  #print(bio.sto.corrected)
  bio.sto.corrected<-mean( bio.sto.corrected)
  # cat('\nmean of sum after adjustement\n')
  #print(bio.sto.corrected)

  bio.deter<-sum(exp(c(mu1,mu2))*weight)

  return(list(dterm=bio.deter,stoch=bio.sto,stoch.corr=bio.sto.corrected))
}

sd1<-seq(0.1,1.0,0.05)
do.it<-function(i){neafc(sd1=sd1[i],mu2Fac=1,var2Fac=1,weight=c(1,1)) }
res<-lapply(1:length(sd1),do.it)

res<-matrix(unlist(res),ncol=3,byrow=T)
matplot(sd1,res,ylab='Absolute values') #abs values

matplot(sd1,res/rep(res[,1],each=3),ylab='Relativ to deterministic') #relative to deterministic values