#################################################################################
# Monte Carlo method to calculate Multi-Species Indices (MSI)
# CBS (Statistics Netherlands), july 2016
# 	- generate MSIs with confidence interval, accounting for sampling error
#   - baseyear for simulated MSIs is flexible
#   - baseyear for plots can be set to 100 for index or for smoothed trend value
#	  - linear trend classification
#   - smoothed trend classification
#   - compare linear trends before and after changepoint
#   - assess overall change and change in last years
#   - truncation of low indices (and their SE set to 0)
#################################################################################

#################################################################################
# Input file has at least these 4 columns:
#   - species (species code)
#   - year
#   - index (base year has index 100; anny year can be the base year)
#   - se (standard error of index; base year has SE zero)
#################################################################################

# Data import and specificatons
setwd("E:/MSI/data") # the path for input and output files.
rdata <- read.table("farmland.txt", header=TRUE) # name of input file.
jobname <- as.character ("farmland") # generic name for output files.
nsim <- 1000		    # number of Monte Carlo simulations.
SEbaseyear <- 1990 # desired year to set MSI to 1000 and SE to 0; usually the first year of the time series.
plotbaseyear <- 1990 # desired year to set to 100 in plots.
index_smooth <- "SMOOTH" # INDEX / SMOOTH: "INDEX" will cause the index in plotbaseyear set to 100; "SMOOTH" will set the smoothed trend value in the plotbaseyear to 100.
lastyears <- 10		# last X years of time series for which separate trend (short-term trends) should be calculated.
maxCV <- 3       # maximum allowed mean Coefficient of Variation (CV) of species indices (0.5 = 50%). Species with higher mean CV are excluded.
changepoint <- 2000 # compare trends before and after this year.
truncfac <- 10    # truncation factor (=max year-to-year index ratio). Default for Living Planet Index = 10.
TRUNC <- 1 # set all indices below TRUNC to this value and their SE to 0. TRUNC <- 0 means no truncation.

# set names of output files
jobnameRESULTS <- paste (jobname, "_RESULTS.csv", sep = "")
jobnameTRENDS <- paste (jobname, "_TRENDS.csv", sep = "")
jobnameSIMTRENDS <- paste (jobname, "_SIMTRENDS.csv", sep = "")
jobnameGRAPH <- paste (jobname, "_GRAPH.jpg", sep = "")

# Define and calculate model parameters
species <- rdata[,"species"]
year <- rdata[,"year"]
index <- rdata[,"index"]
se <- rdata[,"se"]
uspecies <- sort(unique(species))
nspecies <- length(uspecies)
uyear <- sort(unique(year))
nyear <- length(unique(year))
meanindex <- tapply(index, species, mean, na.rm=TRUE)/100
mnindex1 <- as.data.frame(rep(meanindex,each=nyear))
minyear <- min(year)
maxyear <- max(year)
plotbaseyear <- plotbaseyear-minyear+1
baseyear <- max(1,SEbaseyear-minyear+1)
lastyears <- min(lastyears,nyear)
INP <- data.frame(cbind(species, year, index, se))
SPEC <- as.matrix(species)
species <- sort(rep(uspecies,each=nyear))
year <- rep(uyear,nspecies)
INP1 <- data.frame(cbind(species,year))
INP2 <- merge(INP, INP1, by=c("species","year"), sort=TRUE, all=TRUE)

# Calculate and plot mean CV for indices per species
CVtemp <- INP2[INP2$index >= 10, ] # select records with index >= 10 for CV calculation
CVtemp <- CVtemp[!is.na(CVtemp$index), ] # reject missing values
CV1 <- CVtemp$se/CVtemp$index
CV1[CV1== 0] <- NA
CV1[CV1== Inf] <- NA
species2 <- CVtemp$species
mnCV <- tapply(CV1, species2, mean, na.rm=TRUE)
plot(uspecies, mnCV, type="p", pch=19, col="black", main=jobname, xlab="species code", ylab="meanCV")
CV <- as.data.frame(rep(mnCV, each=nyear))

# replace small indices by TRUNC and their SE by zero
INP3 <- data.frame(cbind(species, year, CV, mnindex1))
colnames(INP3) <- c("species","year", "CV", "mnindex1")
INP4 <- merge(INP2, INP3, by=c("species","year"), sort=FALSE, all=TRUE)
INP5 <- subset(INP4, CV < maxCV, select = c(species, year, index, se, mnindex1))
INP5$index[INP5$index < TRUNC & !is.na(INP5$index)] <- TRUNC
INP5$se[INP5$index == TRUNC & !is.na(INP5$index)] <- 0

# reset parameters
index <- as.vector(INP5["index"])
se <- as.vector(INP5["se"])
nobs <- NROW(INP5)
uspecies <- sort(unique(INP5$species))
nspecies <- length(uspecies)
year <- rep(uyear, nspecies)

# Transform indices and standard deviations to log scale (Delta method)
LNindex <- as.vector(log(index))
LNse <- as.vector(se/index)

# Monte Carlo simulations of species indices
MC <- matrix(NA, nobs, nsim)
for (s in 1:nsim) {
       for (o in 1:nobs) {
       MC[o,s] <- rnorm(1, LNindex[o,1], LNse[o,1])
       }
       }
MC[MC < log(TRUNC)] <- log(TRUNC)

# impute missing values using chain method
CHAIN <- matrix(NA, nobs, nsim)
for (s in 1:nsim ){
  for (o in 1:nobs-1) {
    CHAIN[o,s] <- MC[o+1,s]-MC[o,s]
  }
  for (sp in 1:nspecies) {
    CHAIN[sp*nyear,s] <- NA
  }
}

CHAIN[CHAIN > log(truncfac)] <- log(truncfac)
CHAIN[CHAIN < log(1/truncfac)] <- log(1/truncfac)

mnCHAIN <- matrix(NA, nyear, nsim)
for (s in 1:nsim) {
  mnCHAIN[,s] <- tapply(CHAIN[,s], year, mean, na.rm=TRUE)
}

simMSI <- matrix(NA, nyear, nsim)
bmin <- baseyear-1
bplus <- baseyear+1
for (s in 1:nsim){
simMSI[baseyear,s] <- log(100)
  for (y in bmin:min(bmin,1)){
    simMSI[y,s] <- simMSI[y+1,s] - mnCHAIN[y,s]
  }
  for (y in min(bplus,nyear):nyear){
    simMSI[y,s] <- simMSI[y-1,s] + mnCHAIN[y-1,s]
  }
}

# calculate MSI and SE
meanMSI <- array(NA, dim=c(1, nyear))
stdevMSI <- array(NA, dim=c(1, nyear))
for (y in 1:nyear) {
  meanMSI[y] <- mean(simMSI[y,])
  stdevMSI[y] <- sd(simMSI[y,])
}

# Monte Carlo simulations of MSI (for trend calculation)
simMSI <- matrix(NA, nyear, nsim)
for (s in 1:nsim) {
  for (y in 1:nyear) {
    simMSI[y,s] <- rnorm(1, meanMSI[y], stdevMSI[y])
  }
}

# Back-transformation of MSI to index scale (Delta method)
meanMSI <- array(NA, dim=c(1, nyear))
stdevMSI <- array(NA, dim=c(1, nyear))
for (y in 1:nyear) {
  meanMSI[y] <- round(exp(mean(simMSI[y,])), digits=2)
  stdevMSI[y] <- round(sd(simMSI[y,])*meanMSI[y], digits=2)
}

# Confidence interval of MSI on index-scale
CI <- array(NA, dim=c(nyear, 2))
for (y in 1:nyear){
  CI[y,1] <- exp(mean(simMSI[y,])-1.96*sd(simMSI[y,]))
  CI[y,2] <- exp(mean(simMSI[y,])+1.96*sd(simMSI[y,]))
}

# Overall linear trend per simulation
lrMSI <- array(NA, dim=c(2, nsim))
for (s in 1:nsim) {
  summ <- summary(lm(simMSI[,s] ~ uyear, na.action=na.exclude))
  lrMSI[1,s] <- round(summ$coefficients[2], digits=4)	# coeff[2] = slope
  lrMSI[2,s] <- round(summ$coefficients[2,2], digits=4) 	# coeff[2,2] = se of slope
}

SLOPE <- mean(lrMSI[1,])
sdSLOPE <- sd(as.vector(lrMSI[1,]))

# Back transform trends to index scale:
SLOPE_mult <- round(exp(SLOPE), digits=4)
sdSLOPE_mult <- round(sdSLOPE*SLOPE_mult, digits=4)

SLOPE
sdSLOPE
SLOPE_mult
sdSLOPE_mult

# Overall trend classification (see Soldaat et al. 2007 (J. Ornithol. DOI 10.1007/s10336-007-0176-7))
TrendClass <- "X"
if (SLOPE_mult - 1.96*sdSLOPE_mult > 1.05) {TrendClass <- "strong increase" } else
  if (SLOPE_mult + 1.96*sdSLOPE_mult < 0.95) { TrendClass <- "steep decline" } else
    if (SLOPE_mult - 1.96*sdSLOPE_mult > 1.00) { TrendClass <- "moderate increase"} else
      if (SLOPE_mult + 1.96*sdSLOPE_mult < 1.00) { TrendClass <- "moderate decline" } else
        if ((SLOPE_mult - 1.96*sdSLOPE_mult - 0.95)*(1.05 - SLOPE_mult + 1.96*sdSLOPE_mult) < 0.00) { TrendClass <- "uncertain"} else
          if ((SLOPE_mult + 1.96*sdSLOPE_mult) - (SLOPE_mult - 1.96*sdSLOPE_mult) > 0.10) { TrendClass <- "uncertain" } else
          {TrendClass <- "stable"}
TrendClass

# Short term linear trend per simulation
lrMSI_short <- array(NA, dim=c(2, nsim))
lyear <- c((maxyear-lastyears+1):maxyear)
simMSI_short <- array(NA, dim=c(lastyears, nsim))
for (s in 1:nsim) {
  for (y in 1:lastyears) {
    simMSI_short[y, s] <- simMSI[(nyear-lastyears+y),s]
  }
}
for (s in 1:nsim) {
  summ <- summary(lm(simMSI_short[,s] ~ lyear, na.action=na.exclude))
  lrMSI_short[1,s] <- round(summ$coefficients[2], digits=4)	# coeff[2] = slope
  lrMSI_short[2,s] <- round(summ$coefficients[2,2], digits=4) 	# coeff[2,2] = sd of slope
}

SLOPE_short <- mean(lrMSI_short[1,])
sdSLOPE_short <- sd(as.vector(lrMSI_short[1,]))

# Back-transform short-term trends to index scale
SLOPE_short_mult <- round(exp(SLOPE_short), digits=4)
sdSLOPE_short_mult <- round(sdSLOPE_short*SLOPE_short_mult, digits=4)

SLOPE_short
sdSLOPE_short
SLOPE_short_mult
sdSLOPE_short_mult

# Short term trend classification (Soldaat et al. 2007 (J. Ornithol. DOI 10.1007/s10336-007-0176-7))
TrendClass_short <- "X"
if (SLOPE_short_mult - 1.96*sdSLOPE_short_mult > 1.05) {TrendClass_short <- "strong increase" } else
  if (SLOPE_short_mult + 1.96*sdSLOPE_short_mult < 0.95) { TrendClass_short <- "steep decline" } else
    if (SLOPE_short_mult - 1.96*sdSLOPE_short_mult > 1.00) { TrendClass_short <- "moderate increase"} else
      if (SLOPE_short_mult + 1.96*sdSLOPE_short_mult < 1.00) { TrendClass_short <- "moderate decline" } else
        if ((SLOPE_short_mult - 1.96*sdSLOPE_short_mult - 0.95)*(1.05 - SLOPE_short_mult + 1.96*sdSLOPE_short_mult) < 0.00) { TrendClass_short <- "uncertain"} else
          if ((SLOPE_short_mult + 1.96*sdSLOPE_short_mult) - (SLOPE_short_mult - 1.96*sdSLOPE_short_mult) > 0.10) { TrendClass_short <- "uncertain" } else
          {TrendClass_short <- "stable"}
TrendClass_short

# linear trend before changepoint per simulation
lrMSI_before <- array(NA, dim=c(2, nsim))
byear <- c(minyear:changepoint)
byears <- length(byear)
simMSI_before <- array(NA, dim=c(byears, nsim))
for (s in 1:nsim) {
  for (y in 1:byears) {
    simMSI_before[y, s] <- simMSI[y,s]
  }
}
for (s in 1:nsim) {
  summb <- summary(lm(simMSI_before[,s] ~ byear, na.action=na.exclude))
  lrMSI_before[1,s] <- round(summb$coefficients[2], digits=4)  # coeff[2] = slope
  lrMSI_before[2,s] <- round(summb$coefficients[2,2], digits=4) 	# coeff[2,2] = sd of slope
}
SLOPE_before <- round(mean(lrMSI_before[1,]), digits=5)
sdSLOPE_before <- round(sd(as.vector(lrMSI_before[1,])), digits=5)

# Back-transform trends before changepoint to index scale
SLOPE_before_mult <- round(exp(SLOPE_before), digits=4)
sdSLOPE_before_mult <- round(sdSLOPE_before*SLOPE_before_mult, digits=4)

SLOPE_before
sdSLOPE_before
SLOPE_before_mult
sdSLOPE_before_mult

# Trend classification before changepoint
TrendClass_before <- "X"
if (SLOPE_before_mult - 1.96*sdSLOPE_before_mult > 1.05) {TrendClass_before <- "strong increase" } else
  if (SLOPE_before_mult + 1.96*sdSLOPE_before_mult < 0.95) { TrendClass_before <- "steep decline" } else
    if (SLOPE_before_mult - 1.96*sdSLOPE_before_mult > 1.00) { TrendClass_before <- "moderate increase"} else
      if (SLOPE_before_mult + 1.96*sdSLOPE_before_mult < 1.00) { TrendClass_before <- "moderate decline" } else
        if ((SLOPE_before_mult - 1.96*sdSLOPE_before_mult - 0.95)*(1.05 - SLOPE_before_mult + 1.96*sdSLOPE_before_mult) < 0.00) { TrendClass_before <- "uncertain"} else
          if ((SLOPE_before_mult + 1.96*sdSLOPE_before_mult) - (SLOPE_before_mult - 1.96*sdSLOPE_before_mult) > 0.10) { TrendClass_before <- "uncertain" } else
          {TrendClass_before <- "stable"}
TrendClass_before

# linear trend after changepoint per simulation
lrMSI_after <- array(NA, dim=c(2, nsim))
ayear <- c(changepoint:maxyear)
ayears <- length(ayear)
simMSI_after <- array(NA, dim=c(ayears, nsim))
for (s in 1:nsim) {
  for (y in 1:ayears) {
    simMSI_after[y, s] <- simMSI[(byears-1+y),s]
  }
}
for (s in 1:nsim) {
  summa <- summary(lm(simMSI_after[,s] ~ ayear, na.action=na.exclude))
  lrMSI_after[1,s] <- round(summa$coefficients[2], digits=4)  # coeff[2] = slope
  lrMSI_after[2,s] <- round(summa$coefficients[2,2], digits=4)   # coeff[2,2] = sd of slope
}
SLOPE_after <- round(mean(lrMSI_after[1,]), digits=5)
sdSLOPE_after <- round(sd(as.vector(lrMSI_after[1,])),digits=5)

# Trend classification before changepoint
SLOPE_after_mult <- round(exp(SLOPE_after), digits=4)
sdSLOPE_after_mult <- round(sdSLOPE_after*SLOPE_after_mult, digits=4)

SLOPE_after
sdSLOPE_after
SLOPE_after_mult
sdSLOPE_after_mult

# Trend classification after changepoint
TrendClass_after <- "X"
if (SLOPE_after_mult - 1.96*sdSLOPE_after_mult > 1.05) {TrendClass_after <- "strong increase" } else
  if (SLOPE_after_mult + 1.96*sdSLOPE_after_mult < 0.95) { TrendClass_after <- "steep decline" } else
    if (SLOPE_after_mult - 1.96*sdSLOPE_after_mult > 1.00) { TrendClass_after <- "moderate increase"} else
      if (SLOPE_after_mult + 1.96*sdSLOPE_after_mult < 1.00) { TrendClass_after <- "moderate decline" } else
        if ((SLOPE_after_mult - 1.96*sdSLOPE_after_mult - 0.95)*(1.05 - SLOPE_after_mult + 1.96*sdSLOPE_after_mult) < 0.00) { TrendClass_after <- "uncertain"} else
          if ((SLOPE_after_mult + 1.96*sdSLOPE_after_mult) - (SLOPE_after_mult - 1.96*sdSLOPE_after_mult) > 0.10) { TrendClass_after <- "uncertain" } else
          {TrendClass_after <- "stable"}
TrendClass_after

# compare linear trends before and after changepoint
compare <- array(NA, dim=c(nsim, 1))
for (s in 1:nsim){
  compare[s,1] <- lrMSI_after[1,s]-lrMSI_before[1,s]
}
meanTrendDiff <- mean(compare)
seTrendDiff <- sd(as.vector(compare))
significance <- NA
if (abs(meanTrendDiff)-2.58*seTrendDiff > 0) { significance <- "p<0.01" } else
  if (abs(meanTrendDiff)-1.96*seTrendDiff > 0) { significance <- "p<0.05" } else
    {significance <- "n.s."}

# Smoothing
# span = 0.75 is default in R and usually fits well. But trying other values may give better results, depending on your data
loessMSI <- array(NA, dim=c(nyear, nsim))
Diff <- array(NA, dim=c(nyear, nsim))
for (s in 1:nsim) {
  smooth <- predict(loess(simMSI[,s]~uyear, span=0.75, degree=2, na.action=na.exclude),
                    data.frame(uyear), se=TRUE)
  loessMSI[,s] <- round(smooth$fit, digits=4)

  for (y in 1:nyear) {
    Diff[y,s] <- loessMSI[nyear,s] - loessMSI[y,s]
  }
}

# calculate overall change (first year set to 100) and change in last years
pct_CHANGE <- array(NA, dim=c(nsim, 2))
for (s in 1:nsim) {
pct_CHANGE[s,1] <- exp(loessMSI[nyear,s])*100/exp(loessMSI[1,s])-100
pct_CHANGE[s,2] <- exp(loessMSI[nyear,s])*100/exp(loessMSI[1,s])-exp(loessMSI[(nyear-lastyears+1),s])*100/exp(loessMSI[1,s])
}
pct_CHANGE_long <- round(mean(pct_CHANGE[,1]),digits=3)
sd_pct_CHANGE_long <- round(sd(pct_CHANGE[,1]), digits=3)
pct_CHANGE_long
sd_pct_CHANGE_long
pct_CHANGE_short <- round(mean(pct_CHANGE[,2]),digits=3)
sd_pct_CHANGE_short <- round(sd(pct_CHANGE[,2]), digits=3)
pct_CHANGE_short
sd_pct_CHANGE_short

# significance of percentage change
significance_PCT_long <- NA
if (abs(pct_CHANGE_long)-2.58*sd_pct_CHANGE_long > 0) { significance_PCT_long <- "p<0.01" } else
  if (abs(pct_CHANGE_long)-1.96*sd_pct_CHANGE_long > 0) { significance_PCT_long <- "p<0.05" } else
  {significance_PCT_long <- "n.s."}

significance_PCT_short <- NA
if (abs(pct_CHANGE_short)-2.58*sd_pct_CHANGE_short > 0) { significance_PCT_short <- "p<0.01" } else
  if (abs(pct_CHANGE_short)-1.96*sd_pct_CHANGE_short > 0) { significance_PCT_short <- "p<0.05" } else
  {significance_PCT_short <- "n.s."}

# create output for flexible trend estimates
smoothMSI <- array(NA, dim=c(nyear, 13))
for (y in 1:nyear) {
  smoothMSI[y,1] <- round(mean(loessMSI[y,]), digits=4) # smooth MSI on log scale
  smoothMSI[y,2] <- round(sd(loessMSI[y,]), digits=4) # SE of smooth MSI on log scale
  smoothMSI[y,3] <- round(mean(Diff[y,]), digits=4) # Difference smooth MSI with last year on log scale
  smoothMSI[y,4] <- round(sd(Diff[y,]), digits=4) # SE of difference with ast year
  smoothMSI[y,12] <- round(smoothMSI[y,1]-1.96*smoothMSI[y,2], digits=2) # lower CI of smooth MSI on log scale
  smoothMSI[y,13] <- round(smoothMSI[y,1]+1.96*smoothMSI[y,2], digits=2) # upper CI of smooth MSI on log scale
}

# Trendclassification, based on smoothed trends (Soldaat et al. 2007 (J. Ornithol. DOI 10.1007/s10336-007-0176-7))
for (y in 1:nyear) {
  smoothMSI[y,5] <- round(smoothMSI[nyear,1]/smoothMSI[y,1], digits=4)		# = TCR
  smoothMSI[y,6] <- round(smoothMSI[y,5]-1.96*(smoothMSI[y,4]/smoothMSI[y,1]), digits=4)	# = CI- TCR
  smoothMSI[y,7] <- round(smoothMSI[y,5]+1.96*(smoothMSI[y,4]/smoothMSI[y,1]), digits=4)	# = CI+ TCR
  smoothMSI[y,8] <- round(exp(log(smoothMSI[y,5])/(nyear-y)), digits=4)		# = YCR
  smoothMSI[y,9] <- round(exp(log(smoothMSI[y,6])/(nyear-y)), digits=4)		# = CI- YCR
  smoothMSI[y,10] <- round(exp(log(smoothMSI[y,7])/(nyear-y)), digits=4)		# = CI+ YCR
}

for (y in 1:(nyear-1)) {
  if (smoothMSI[y,9] > 1.05) {smoothMSI[y,11] <- 1} else
    if (smoothMSI[y,10] < 0.95) {smoothMSI[y,11] <- 6} else
      if (smoothMSI[y,9] > 1.00) {smoothMSI[y,11] <- 2} else
        if (smoothMSI[y,10] < 1.00) {smoothMSI[y,11] <- 5} else
          if ((smoothMSI[y,9] - 0.95)*(1.05 - smoothMSI[y,10]) < 0.00) {smoothMSI[y,11] <- 3} else
            if ((smoothMSI[y,10]) - (smoothMSI[y,9]) > 0.10) {smoothMSI[y,11] <- 3} else
            {smoothMSI[y,11] <- 4}
}

TrendClass_flex <- matrix(NA, nrow= nyear, ncol=1)
for (y in 1:(nyear-1)) {
  if (smoothMSI[y,9] > 1.05) {TrendClass_flex[y] <- "strong_increase" } else
    if (smoothMSI[y,10] < 0.95) { TrendClass_flex[y] <- "steep_decline" } else
      if (smoothMSI[y,9] > 1.00) { TrendClass_flex[y] <- "moderate_increase"} else
        if (smoothMSI[y,10] < 1.00) { TrendClass_flex[y] <- "moderate_decline" } else
          if ((smoothMSI[y,9] - 0.95)*(1.05 - smoothMSI[y,10]) < 0.00) { TrendClass_flex[y] <- "uncertain"} else
            if ((smoothMSI[y,10]) - (smoothMSI[y,9]) > 0.10) { TrendClass_flex[y] <- "uncertain" } else
            {TrendClass_flex[y] <- "stable"}
}

# rescaling (for presentation of plot)
rescale <- NA
 if (index_smooth =="INDEX") {rescale <- 100/meanMSI[plotbaseyear]} else
  if (index_smooth =="SMOOTH") {rescale <- 100/exp(smoothMSI[plotbaseyear,1])} else
   {rescale <- NA}
simMSImean <- round(as.vector(rescale*meanMSI), digits=2)
simMSIsd <- round(as.vector(rescale*stdevMSI), digits=2)
uppCI_MSI <- round(rescale*CI[,2], digits=2)
lowCI_MSI <- round(rescale*CI[,1], digits=2)
trend_flex <- round(rescale*exp(smoothMSI[,1]), digits=2)
lowCI_trend_flex <- round(rescale*exp(smoothMSI[,12]), digits=2)
uppCI_trend_flex <- round(rescale* exp(smoothMSI[,13]), digits=2)

# create output file for MSI + smoothed trend
RES <- as.data.frame(cbind(uyear, simMSImean, simMSIsd, lowCI_MSI, uppCI_MSI, trend_flex, lowCI_trend_flex, uppCI_trend_flex))
RES$trend_class <- TrendClass_flex
names(RES) <- c("year", "MSI", "sd_MSI", "lower_CL_MSI", "upper_CL_MSI", "Trend", "lower_CL_trend", "upper_CL_trend", "trend_class")
write.csv2(RES, file=jobnameRESULTS, row.names=FALSE, quote=FALSE)

# create output file with all linear trends
SIMTRENDS <- t(rbind(lrMSI[1,],lrMSI_short[1,]))
write.csv2(SIMTRENDS, file=jobnameSIMTRENDS, row.names=FALSE)

# create output for linear trend estimates and % change
rownames <- rbind("overall trend","SE overall trend", "trend last years", "SE trend last years", "changepoint",
                  "trend before changepoint", "SE trend before changepoint", "trend after changepoint", "SE trend after changepoint",
                  "% change", "SE % change", "% change last years", "SE % change last years")
output <- cbind(SLOPE_mult, sdSLOPE_mult, SLOPE_short_mult, sdSLOPE_short_mult,
                changepoint, SLOPE_before_mult, sdSLOPE_before_mult, SLOPE_after_mult, sdSLOPE_after_mult,
                pct_CHANGE_long, sd_pct_CHANGE_long, pct_CHANGE_short, sd_pct_CHANGE_short)
TRENDS <- as.data.frame(t(output), row.names=rownames)
TRENDS$significance <- ""
TRENDS["changepoint", "significance"] <- significance
TRENDS[1, "significance"] <- TrendClass
TRENDS[3, "significance"] <- TrendClass_short
TRENDS[6, "significance"] <- TrendClass_before
TRENDS[8, "significance"] <- TrendClass_after
TRENDS[10, "significance"] <- significance_PCT_long
TRENDS[12, "significance"] <- significance_PCT_short
names(TRENDS) <- c("value", "significance")
write.csv2(TRENDS, file=jobnameTRENDS)

# create output file with plot
maxy <- max(uppCI_MSI) + 10
jpeg(filename = jobnameGRAPH, width = 480, height = 480,units = "px", pointsize = 12, bg = "white", res = NA, restoreConsole = TRUE)
legend1 <- paste(nspecies, "species")
legend2 <- paste(TrendClass)
legend3 <- paste("last", lastyears, "years:", TrendClass_short)
plot(uyear, simMSImean, type="p", pch=19, col="black", main=jobname, xlab="", ylab="Index", xlim = c(minyear, maxyear), ylim = c(0,maxy))
text(minyear, 40, legend1, adj=0, cex=0.8)
text(minyear, 25, legend2, adj=0, cex=0.8)
text(minyear, 10, legend3, adj=0, cex=0.8)
lines(uyear, rescale*exp(smoothMSI[,1]), lty=1, col="black")
lines(uyear, rescale*exp(smoothMSI[,12]), lty=3, col="black")
lines(uyear, rescale*exp(smoothMSI[,13]), lty=3, col="black")
#arrows(uyear, simMSImean, uyear, (simMSImean-simMSIsd), angle = 90, code = 3, length=0)
#arrows(uyear, simMSImean, uyear, (simMSImean+simMSIsd), angle = 90, code = 3, length=0)
dev.off()

# plot to screen
maxy <- max(uppCI_MSI) + 10
plot(uyear, simMSImean, type="p", pch=19, col="black", main=jobname, xlab="", ylab="Index", xlim = c(minyear, maxyear), ylim = c(0,maxy))
text(minyear, 40, legend1, adj=0, cex=0.8)
text(minyear, 25, legend2, adj=0, cex=0.8)
text(minyear, 10, legend3, adj=0, cex=0.8)
lines(uyear, rescale*exp(smoothMSI[,1]), lty=1, col="black")
lines(uyear, rescale*exp(smoothMSI[,12]), lty=3, col="black")
lines(uyear, rescale*exp(smoothMSI[,13]), lty=3, col="black")
arrows(uyear, simMSImean, uyear, (simMSImean-simMSIsd), angle = 90, code = 3, length=0)
arrows(uyear, simMSImean, uyear, (simMSImean+simMSIsd), angle = 90, code = 3, length=0)


#----------------------------------------------------------------------------------------------------------
# Name:         EV_RP.r
# Purpose:      This main function estimates return periods using the block maxima/minima approach
#				on gridded data. All external modules are called within this script.
# Author:       Francesco Tonini
# Email: 	    f_tonini@hotmail.com
# Created:      11/1/2009
# Copyright:    (c) 2009 by Francesco Tonini
# License:    	GNU General Public License (GPL)
# Software:     Tested successfully using R-2.14.0 64-bit version(http://www.r-project.org/)
#-------------------------------------------------------------------------------------------

##Define the main working directory
##Always use either / or \\ to specify the path
mainDir <- 'C:/Temp'
#mainDir <- '~/Desktop/Temp'  #if you are on a MacOSX environment

##Let's set the working directory
setwd(file.path(mainDir))

##Define the subdirectory(ies) where your output files will be saved
subDir <- 'Output'

##Create a physical copy of the subdirectory folder(s) where you save your output
##If the directory already exists it gives a warning, BUT we can suppress it using showWarnings = FALSE
dir.create(file.path(mainDir, subDir), showWarnings = FALSE)

##Use an external source file w/ all modules (functions) used within this script.
##Use FULL PATH if source file is not in the same folder w/ this script
source('myEVfunctions.r')

##Path to folders in which you want to save all your output files
workdir_Output <- file.path(mainDir, subDir)

##Load all required libraries
print('Loading required libraries...')
load.packages()

##Let's set the simulation parameters:

##Alamata.tot is the dataset we built for the Alamata woreda, Tigray (Ethiopia).
##This can be replaced by an entire region or country (the bigger the extent, the more the code will take to finish)
dataset <- Alamata.tot

##Number of pixels within the chosen dataset
npixels <- length(unique(dataset$id))

##Let's specify the return level(s) for which we want to calculate the associated probability of exceedance
##(i.e. return period)
return_levels <- c(-0.15,-0.05)

##Let's create a dataframe to store the parameters of the GEV distributions and their confidence intervals
tab.parameters <- data.frame(matrix(NA,npixels,11))
colnames(tab.parameters) <- c('id.pix','gum.flag','mu','sig','shp','mu.inf','mu.sup','sig.inf','sig.sup','shp.inf','shp.sup')

##Let's create some labels for the return levels estimates and their upper and lower confidence intervals
RP_labels <- paste('prob_',return_levels,sep='')

##Let's create a dataframe to store return levels estimates and their upper and lower confidence intervals
tab.prob <- data.frame(matrix(NA, npixels, length(return_levels) + 1))
colnames(tab.prob) <- c("id.pix",RP_labels)

##Create an empty vector in which we would save the pixel IDs
##whose GEV/Gumbel parameter estimates did not converge
px.idWarning <- c()

######################################################################
##Flagged values (in DN) by VITO for SPOT VGT S10
#251=missing, 252=cloud, 253=snow, 254=sea, 255=background
#Converted to ADVI values using: ADVI = -1.25 + 0.01 * DN
#1.26=missing, 1.27=cloud, 1.28=snow, 1.29=sea, 1.30=background
######################################################################

##Create a Tk window element as a container
root <- tktoplevel()
tktitle(root) <- 'Extreme Value Analysis'
##Define the label of a progress bar
lab <- tk2label(root)
##Define the length and create a progress bar
pb <- tk2progress(root, length = 300)
#Configure
tkconfigure(pb, value=0, maximum = npixels-1)
##Pack all labels and progress bars inside the Tk container
tkpack(lab)
tkpack(pb)
##Refresh the Tk window
tcl('update')

##LOOP through each pixel, create its time series of maxima/minima,
##estimate GEV parameters and return levels
for (pix in 1:npixels){

	#update progress bar according to the pixel currently being processed
	tkconfigure(lab, text = paste('Pixel',pix,'Out of',npixels, '    (',round(pix/npixels*100),'% )'))
    tkconfigure(pb, value = pix - 1)
	tcl('update')

	##Slice the main dataset by extracting only historical records for the current pixel
	pix.dataset <- dataset[dataset$id == pix,]

	##This section was developed specifically for the case of having dekadal (1 dekad = 10days)
	##time series and extracting monthly (1 month = 3 dekads) MINIMA.
	##NOTE: If you are interested in taking monthly MAXIMA just use FUN = max
	##NOTE2: If you wanna take YEARLY minima/maxima then you should use:
	##etiq <- factor(pix.dataset$year) ...BUT make sure the time series is longer than at least 30 years
	##otherwise the GEV estimates will not be reliable, as well as the return level estimates.

	##Identify the 'dekadal' variable in the dataset
	Dekads <- pix.dataset$decads

	##Identify the 'Year' variable in the dataset
	Years <- pix.dataset$year

	##Identify the 'Value' variable in the dataset
	Values <- pix.dataset$val

	##Let's store the labels (as factors) of all existing dekad-year pairs (regardless of the dekad number)
	etiq <- factor( paste(substring(Dekads,1,3), Years, sep='') )

	##Let's extract the minima (use FUN = max if you want maxima) for each  unique level of the factor
	monthExtract <- tapply(Values, etiq, FUN = min)

	##Since values in 'monthExtract' have an alphabetical order relative to the month (Apr1998,Apr1999,Apr2000,etc.)
	##we need to sort them so that the series of values follows the real time line

	##If your are taking YEARLY minima/maxima skip (1-2-3) and use only (4). This, because
	##years already have a natural ascending order:

	#(1) by taking unique factor levels, we have a natural time line order
	UniqueEtiq <- unique(etiq)

	#(2) let's now match the correct ordered position of unique factors with our monthly values
	PosIdx <- match(as.character(UniqueEtiq), names(monthExtract))

	#(3) now that we got all the correct positions, let's put all our values in a numerical vector. This
	#is the time series to be used with extreme value models
	px.min <- as.vector(monthExtract[PosIdx])

	#(4) Uncomment the next line and use it to build your YEARLY maxima/minima time series
	#px.min <- as.vector(monthExtract)

	#If the pixel is coded as 1.29=sea or 1.30=background we simply put 999
	if ( any(px.min == 1.29) | any(px.min == 1.30) ) {
		tab.parameters[pix,] <- 999
		tab.prob[pix,] <- 999
		next
	}

	#Let's now look at values 1.26=missing, 1.27=cloud, 1.28=snow. They will need to be removed from the
	#numerical vector, otherwise the extreme models will be biased. Since in our study area there are very few
	#or no cloud values, snow, or missing, we can either remove them from the numerical vector, or
	#replace them with a simple average between N previous/following values.
	Flags <- c(1.26, 1.27, 1.28)

	#replace flagged values with NAs (if there are any)
	px.min[ px.min %in% Flags] <- NA

	#if there are flagged values, count them and see if their number exceeds 15% of the total time series
	if (any(is.na(px.min))) {
		NA.sum <- sum(is.na(px.min))
		#print a warning on screen to let the user know that model estimates may be biased
		if (NA.sum / length(px.min) * 100 >= 15) cat('\nATTENTION: More than 15% of values are flagged.\nModel estimates may be biased')
	}

	#override the original time series by removing any NA (flagged) values
	px.min <- px.min[!is.na(px.min)]

	#estimate GEV model (if you are working with maxima, DO NOT use the '-' sign
	minimi.gev <- gev.fit(-px.min,show=F)
	class(minimi.gev) <- "gev.fit"

	#estimate Gumbel model (if you are working with maxima, DO NOT use the '-' sign
	minimi.gum <- gum.fit(-px.min,show=F)
	class(minimi.gum) <- "gum.fit"

	#If the MLE estimation procedure does not converge, save the pixel ID into
	#the warning vector
	if (minimi.gev$conv > 0 | minimi.gum$conv > 0){

		px.idWarning <- c(px.idWarning, pix)

	}else{

		# Likelihood Ratio method to compare two models
		lik.ratio <- 2 * (minimi.gum$nllh - minimi.gev$nllh)
		pvalue <- (1 - pchisq(lik.ratio,1))

		#if the p-value is not significant (>.05) we use the gumbel distr. (shape parameter --> 0)
		if (pvalue > 0.05){

			#use a variable to flag the use of a gumbel distr.
			gum.flag <- 1
			mu <- minimi.gum$mle[1]  #location
			sig <- minimi.gum$mle[2] #scale
			shp <- 0                 #shape
			#conf. intervals (based ~ N(0,1))
			mu.inf <- minimi.gum$mle[1] - 1.96 * minimi.gum$se[1]
			mu.sup <- minimi.gum$mle[1] + 1.96 * minimi.gum$se[1]
			sig.inf <- minimi.gum$mle[2] - 1.96 * minimi.gum$se[2]
			sig.sup <- minimi.gum$mle[2] + 1.96 * minimi.gum$se[2]
			shp.inf <- 0
			shp.sup <- 0

			#calculate probability of observing the desired return level(s)
			#if you are using MAXIMA, go to the myEVfunctions.r file and adjust
			#the code within the prob_fun(). Follow the comments.
			prob <- prob_fun()

		}else{ #otherwise use the GEV distr.

			#use a variable to flag the NON use of a gumbel distr.
			gum.flag <- 0
			mu <- minimi.gev$mle[1]  #location
			sig <- minimi.gev$mle[2] #scale
			shp <- minimi.gev$mle[3] #shape
			#conf. intervals (based ~ N(0,1))
			mu.inf <- minimi.gev$mle[1] - 1.96 * minimi.gev$se[1]
			mu.sup <- minimi.gev$mle[1] + 1.96 * minimi.gev$se[1]
			sig.inf <- minimi.gev$mle[2] - 1.96 * minimi.gev$se[2]
			sig.sup <- minimi.gev$mle[2] + 1.96 * minimi.gev$se[2]
		    shp.inf <- minimi.gev$mle[3] - 1.96 * minimi.gev$se[3]
			shp.sup <- minimi.gev$mle[3] + 1.96 * minimi.gev$se[3]

			#calculate probability of observing the desired return level(s)
			#if you are using MAXIMA, go to the myEVfunctions.r file and adjust
			#the code within the prob_fun(). Follow the comments.
			prob <- prob_fun()
		}

		#let's store the parameter and probability of exceedance estimates in their respective dataset, for the row
		#corresponding to the current pixel in the LOOP
		tab.parameters[pix,] <- c(pix,gum.flag,round(mu,2),round(sig,2),round(shp,2),round(mu.inf,2),round(mu.sup,2),
								 round(sig.inf,2),round(sig.sup,2),round(shp.inf,2),round(shp.sup,2))

		tab.prob[pix,] <- c(pix, prob)

	}

}

#For those pixels whose return levels could not be computed, use the median of the 4 nearest neighbors
if(length(px.idWarning) > 0) median.interp_RP()

#Save files to output folder
save_files_RP()

tkmessageBox(title = "Message", message = 'Done! Output Files Saved To Folder', icon = "info", type = "ok")
##Suppress the progress bar
tkdestroy(root)


#' ms2mph
#'
#' Conversion speed from meter per second to knots.
#'
#' @param numeric  ms Speed in meter per second.
#' @return
#'
#'
#' @author  Istituto di Biometeorologia Firenze Italy  Alfonso Crisci \email{a.crisci@@ibimet.cnr.it}
#' @keywords  ms2mph
#'
#' @export
#'
#'
#'
#'

ms2mph<-function(ms)
{
  return (ms * 2.23694);
}


model_snowdepthtrans <- function (Sdepth = 0.0,
         Pns = 100.0){
    #'- Name: SnowDepthTrans -Version: 1.0, -Time step: 1
    #'- Description:
    #'            * Title: snow cover depth conversion
    #'            * Author: STICS
    #'            * Reference: doi:http://dx.doi.org/10.1016/j.agrformet.2014.05.002
    #'            * Institution: INRA
    #'            * Abstract: snow cover depth in cm
    #'- inputs:
    #'            * name: Sdepth
    #'                          ** description : snow cover depth Calculation
    #'                          ** inputtype : variable
    #'                          ** variablecategory : state
    #'                          ** datatype : DOUBLE
    #'                          ** default : 0.0
    #'                          ** min : 0.0
    #'                          ** max : 500.0
    #'                          ** unit : m
    #'                          ** uri :
    #'            * name: Pns
    #'                          ** description : density of the new snow
    #'                          ** inputtype : parameter
    #'                          ** parametercategory : constant
    #'                          ** datatype : DOUBLE
    #'                          ** default : 100.0
    #'                          ** min :
    #'                          ** max :
    #'                          ** unit : cm/m
    #'                          ** uri :
    #'- outputs:
    #'            * name: Sdepth_cm
    #'                          ** description : snow cover depth in cm
    #'                          ** variablecategory : state
    #'                          ** datatype : DOUBLE
    #'                          ** min : 0.0
    #'                          ** max : 500.0
    #'                          ** unit : cm
    #'                          ** uri :
    Sdepth_cm <- Sdepth * Pns
    return (list('Sdepth_cm' = Sdepth_cm))
}


path <- '/home/maanitou/programming/fsharp/rufc/Samples/schrodinger/'

content_f <- read.table(paste(path, 'output_schrodinger_forward.txt', sep=""))
content_b <- read.table(paste(path, 'output_schrodinger_backward.txt', sep=""))

factor <- 1000000

Xf <- sapply(content_f, function(x) as.numeric(x)/factor)
Xb <- sapply(content_b, function(x) as.numeric(x)/factor)

par(mfrow=c(1,1))

plot(1:128, Xf[1,], type='l',  xlim=c(0,256), ylim=c(min(Xf),max(Xf)))
lines(1:128, Xf[2,], type='l')
lines(129:256, Xb[1,], type='l')
lines(129:256, Xb[2,], type='l')

for (i in 0:(nrow(Xf)/2-1)) {
  lines(1:128, Xf[2*i+1,], type='l', col='red', xlim=c(0,256), ylim=c(min(Xf),max(Xf)))
  lines(1:128, Xf[2*i+2,], type='l', col='black')
  lines(129:256, Xb[2*i+1,], type='l', col='red')
  lines(129:256, Xb[2*i+2,], type='l', col='black')
  #Sys.sleep(1)
}

#pdf(file="/home/maanitou/Desktop/PAT_2021/project/schrodinger.pdf", width=8, height=4)
par(mfrow=c(3,4), oma=c(0,0,0,0), mar=c(.5,.5,.5,.5))
  for (i in c(seq(0,999,91),999)) {
  #for (i in c(seq(0,4999,455),4999)) {
  #for (i in c(seq(0,1999,182),1999)) {
  plot(1:128, Xf[2*i+1,], type='l', col='black',
    xlim=c(0,256), ylim=c(min(Xf),max(Xf)),
    xlab="", ylab="", xaxt='n', yaxt='n')
  lines(1:128, Xf[2*i+2,], type='l', col='red')
  lines(129:256, Xb[2*i+1,], type='l', col='red')
  lines(129:256, Xb[2*i+2,], type='l', col='black')
  abline(v=128, lty=2)
}
#dev.off()


# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

#' Evaluation of the maximum likelihood using FaST-LMM method
#'
#' Last update: January 11, 2017
#'
#' @author Xiaolei Liu (modified)
#'
#' @param pheno a two-column phenotype matrix
#' @param snp.pool matrix for pseudo QTNs
#' @param X0 covariates matrix
#' @param ncpus number of threads used for parallel computation
#'
#' @return
#' Output: beta - beta effect
#' Output: delta - delta value
#' Output: LL - log-likelihood
#' Output: vg - genetic variance
#' Output: ve - residual variance
#'
`MVP.FaSTLMM.LL` <- function(pheno, snp.pool, X0=NULL, ncpus=2){
    y=pheno
    p=0
    deltaExpStart = -5
    deltaExpEnd = 5
    snp.pool=snp.pool[,]
    if(!is.null(snp.pool)&&var(snp.pool)==0){
        deltaExpStart = 100
        deltaExpEnd = deltaExpStart
    }
    if(is.null(X0)) {
        X0 = matrix(1, nrow(snp.pool), 1)
    }
    X=X0
    #########SVD of X
    K.X.svd <- svd(snp.pool)
    d=K.X.svd$d
    d=d[d>1e-08]
    d=d^2
    U1=K.X.svd$u
    U1=U1[,1:length(d)]
    #handler of single snp
    if(is.null(dim(U1))) U1=matrix(U1,ncol=1)
    n=nrow(U1)
    U1TX=crossprod(U1,X)
    U1TY=crossprod(U1,y)
    yU1TY <- y-U1%*%U1TY
    XU1TX<- X-U1%*%U1TX
    IU = -tcrossprod(U1)
    diag(IU) = rep(1,n) + diag(IU)
    IUX=crossprod(IU,X)
    IUY=crossprod(IU,y)
    #Iteration on the range of delta (-5 to 5 in glog scale)
    delta.range <- seq(deltaExpStart,deltaExpEnd,by=0.1)
    m <- length(delta.range)
    #for (m in seq(deltaExpStart,deltaExpEnd,by=0.1)){
    beta.optimize.parallel <- function(ii){
        #p=p+1
        delta <- exp(delta.range[ii])
        #----------------------------calculate beta-------------------------------------
        #######get beta1
        beta1=0
        for(i in 1:length(d)){
            one=matrix(U1TX[i,], nrow=1)
            beta=crossprod(one,(one/(d[i]+delta)))  #This is not real beta, confusing
            beta1= beta1+beta
        }

        #######get beta2
        beta2=0
        for(i in 1:nrow(U1)){
            one=matrix(IUX[i,], nrow=1)
            beta = crossprod(one)
            beta2= beta2+beta
        }
        beta2<-beta2/delta

        #######get beta3
        beta3=0
        for(i in 1:length(d)){
            one1=matrix(U1TX[i,], nrow=1)
            one2=matrix(U1TY[i,], nrow=1)
            beta=crossprod(one1,(one2/(d[i]+delta)))
            beta3= beta3+beta
        }

        ###########get beta4
        beta4=0
        for(i in 1:nrow(U1)){
            one1=matrix(IUX[i,], nrow=1)
            one2=matrix(IUY[i,], nrow=1)
            beta=crossprod(one1,one2)
            beta4= beta4+beta
        }
        beta4<-beta4/delta

        #######get final beta
        zw1 <- ginv(beta1+beta2)
        #zw1 <- try(solve(beta1+beta2))
        #if(inherits(zw1, "try-error")){
        #zw1 <- ginv(beta1+beta2)
        #}

        zw2=(beta3+beta4)
        beta=crossprod(zw1,zw2)

        #----------------------------calculate LL---------------------------------------
        ####part 1
        part11<-n*log(2*3.14)
        part12<-0
        for(i in 1:length(d)){
            part12_pre=log(d[i]+delta)
            part12= part12+part12_pre
        }
        part13<- (nrow(U1)-length(d))*log(delta)
        part1<- -1/2*(part11+part12+part13)

        ######  part2
        part21<-nrow(U1)
        ######part221

        part221=0
        for(i in 1:length(d)){
            one1=U1TX[i,]
            one2=U1TY[i,]
            part221_pre=(one2-one1%*%beta)^2/(d[i]+delta)
            part221 = part221+part221_pre
        }

        part222=0
        for(i in 1:n){
            one1=XU1TX[i,]
            one2=yU1TY[i,]
            part222_pre=((one2-one1%*%beta)^2)/delta
            part222= part222+part222_pre
        }
        part22<-n*log((1/n)*(part221+part222))
        part2<- -1/2*(part21+part22)

        ################# likihood
        LL<-part1+part2
        part1<-0
        part2<-0

        return(list(beta=beta,delta=delta,LL=LL))
    }
    #} # end of Iteration on the range of delta (-5 to 5 in glog scale)

    llresults <- lapply(1:m, beta.optimize.parallel)

    for(i in 1:m){
        if(i == 1){
            beta.save = llresults[[i]]$beta
            delta.save = llresults[[i]]$delta
            LL.save = llresults[[i]]$LL
        }else{
            if(llresults[[i]]$LL > LL.save){
                beta.save = llresults[[i]]$beta
                delta.save = llresults[[i]]$delta
                LL.save = llresults[[i]]$LL
            }
        }
    }
    #--------------------update with the optimum------------------------------------
    beta=beta.save
    delta=delta.save
    LL=LL.save

    #--------------------calculating Va and Vem-------------------------------------
    #sigma_a1
    sigma_a1=0
    for(i in 1:length(d)){
        one1=matrix(U1TX[i,], ncol=1)
        one2=matrix(U1TY[i,], nrow=1)
        #sigma_a1_pre=(one2-one1%*%beta)^2/(d[i]+delta)
        sigma_a1_pre=(one2-crossprod(one1,beta))^2/(d[i]+delta)
        sigma_a1= sigma_a1+sigma_a1_pre
    }

    ### sigma_a2
    sigma_a2=0

    for(i in 1:nrow(U1)){
        one1=matrix(IUX[i,], ncol=1)
        one2=matrix(IUY[i,], nrow=1)
        #sigma_a2_pre<-(one2-one1%*%beta)^2
        sigma_a2_pre<-(one2-crossprod(one1,beta))^2
        sigma_a2= sigma_a2+sigma_a2_pre
    }

    sigma_a2<-sigma_a2/delta
    sigma_a<- 1/n*(sigma_a1+sigma_a2)
    sigma_e<-delta*sigma_a

    return(list(beta=beta, delta=delta, LL=LL, vg=sigma_a, ve=sigma_e))
}


########################################################
# The MIT License (MIT)
#
# Copyright (c) 2014 Florian D. Schneider & Sonia Kéfi
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
########################################################

### mussel bed model (Guichard et al 2003)

musselbed <- list()
class(musselbed) <- "ca_model"
musselbed$name <- "Mussel Disturbance Model"
musselbed$ref <- "Guichard et al 2003, American Naturalist, Vol. 161, pp. 889–904"
musselbed$states <- c("+", "0", "-")
musselbed$cols <- grayscale(3)
musselbed$parms <- list(
  r = 0.4, # recolonisation of empty sites dependent on local density
  d = 0.9, # probability of disturbance of occupied sites if at least one disturbed site
  delta = 0.01 # intrinsic disturbance rate
)
musselbed$update <- function(x_old, parms_temp, delta = 0.2, subs = 10, timestep = NA) {

  x_new <- x_old

  for(s in 1:subs) {

    parms_temp$rho_plus <- sum(x_old$cells == "+")/(x_old$dim[1]*x_old$dim[2]) # get initial vegetation cover
    parms_temp$localdisturbance <- count(x_old, "-") > 0  # any disturbance in neighborhood?
    parms_temp$localcover <- count(x_old, "+")/4 # any occupied in neighborhood?

    # 2 - drawing random numbers
    rnum <- runif(x_old$dim[1]*x_old$dim[2]) # one random number between 0 and 1 for each cell

    # 3 - setting transition probabilities

    if(parms_temp$rho_plus > 0) {
      recolonisation <- with(parms_temp, (r*localcover)*1/subs)

      disturbance <- with(parms_temp, (delta+d*localdisturbance)*1/subs)
      disturbance[disturbance > 1] <- 1
    } else {
      recolonisation <- 0
      disturbance <- 1
    }

    regeneration <- 1*1/subs

    # check for sum of probabilities to be inferior 1 and superior 0
    if(any(c(recolonisation, disturbance, regeneration) > 1 )) warning(paste("a set probability is exceeding 1 in  time step", timestep, "! decrease number of substeps!!!"))
    if(any(recolonisation < 0)) warning(paste("recolonisation falls below 0 in time step",timestep, "! balance parameters!!!"))
    if(any( disturbance < 0)) warning(paste("disturbance falls below 0 in time step",timestep, "! balance parameters!!!"))
    if(any(regeneration < 0)) warning(paste("regeneration falls below 0 in time step",timestep, "! balance parameters!!!"))

    # 4 - apply transition probabilities

    x_new$cells[which(x_old$cells == "0" & rnum <= recolonisation)] <- "+"
    x_new$cells[which(x_old$cells == "+"  & rnum <= disturbance)] <- "-"
    x_new$cells[which(x_old$cells == "-"   & rnum <= regeneration)] <- "0"

    # 5- store x_new as next x_old

    x_old <- x_new

  }

  ## end of single update call
  return(x_new)
}


[[[[[
   Given an expr to enumerate (program output) pairs,
   we simulate the Turing machine defined this way.
   We assume this r.e. set of programs is prefix free,
   i.e., no extension of a valid program is a valid
   program.  If so, we will carry out this simulation
   in a self-delimiting way, i.e., we won't read any
   unnecessary bits of the program.
]]]]]

define pi

[this is the prefix to put in front of the expr to
enumerate the infinite set of (program output) pairs]

    [graph is an unending expression for (p o) pairs]
    let graph read-exp

    [program read so far; initialize to empty bit string]
    let p nil

    let (look-for p [in] pairs)
       if atom pairs false
       [(add new macro caar -> car car to interpreter?)]
       if = p car car pairs [does first pair start with p?]
          car pairs   [if so, return first pair]
          [otherwise, keep looking]
          (look-for p [in] cdr pairs)

    let (look-for-extension-of p [in] pairs)
       if atom pairs false
       if (is-prefix? p car car pairs)
          true
          (look-for-extension-of p [in] cdr pairs)

    let (is-prefix? p q) [is p a prefix of q?]
       if atom p true
       if atom q false
       if = car p car q
          (is-prefix? cdr p cdr q)
          false

    let (loop t)
       [run for time t expr to generate (program output) pairs]
       [pairs are displayed by graph]
       let pairs debug caddr try debug t graph nil
       let found-it? (look-for p pairs) [found pair with program p?]
       if found-it? cadr found-it? [if so, we have output for p!]
       [(if found-it? isn't false, then it's considered true)]
       [is an extension of p in there?]
       if (look-for-extension-of p [in] pairs)
                  [if so, read another bit of p]
           [then] let p debug append p cons read-bit nil
                     (loop + t 1) [and generate more pairs]
                  [don't read more of p, just generate more pairs]
           [else] (loop + t 1)

    [initialize time t to 0]
    (loop 0)

define      pi
value       ((' (lambda (graph) ((' (lambda (p) ((' (lambda (l
            ook-for) ((' (lambda (look-for-extension-of) ((' (
            lambda (is-prefix?) ((' (lambda (loop) (loop 0)))
            (' (lambda (t) ((' (lambda (pairs) ((' (lambda (fo
            und-it?) (if found-it? (car (cdr found-it?)) (if (
            look-for-extension-of p pairs) ((' (lambda (p) (lo
            op (+ t 1)))) (debug (append p (cons (read-bit) ni
            l)))) (loop (+ t 1)))))) (look-for p pairs)))) (de
            bug (car (cdr (cdr (try (debug t) graph nil)))))))
            )))) (' (lambda (p q) (if (atom p) true (if (atom
            q) false (if (= (car p) (car q)) (is-prefix? (cdr
            p) (cdr q)) false)))))))) (' (lambda (p pairs) (if
             (atom pairs) false (if (is-prefix? p (car (car pa
            irs))) true (look-for-extension-of p (cdr pairs)))
            )))))) (' (lambda (p pairs) (if (atom pairs) false
             (if (= p (car (car pairs))) (car pairs) (look-for
             p (cdr pairs))))))))) nil))) (read-exp))

[graph = (1 0) (01 1) (001 2) (0001 3) (00001 4) etc.]
define graph
    let (loop p n)
  [do!] cons display cons p cons n nil
             (loop  cons 0 p  + 1 n)
    (loop '(1) 0)

define      graph
value       ((' (lambda (loop) (loop (' (1)) 0))) (' (lambda (
            p n) (cons (display (cons p (cons n nil))) (loop (
            cons 0 p) (+ 1 n))))))

[test it!]

try 10 graph nil

expression  (try 10 graph nil)
value       (failure out-of-time (((1) 0) ((0 1) 1) ((0 0 1) 2
            ) ((0 0 0 1) 3) ((0 0 0 0 1) 4) ((0 0 0 0 0 1) 5)
            ((0 0 0 0 0 0 1) 6) ((0 0 0 0 0 0 0 1) 7) ((0 0 0
            0 0 0 0 0 1) 8)))

run-utm-on

append

     bits pi

append

     bits graph

     '(0 0 0 1)

expression  (car (cdr (try no-time-limit (' (eval (read-exp)))
             (append (bits pi) (append (bits graph) (' (0 0 0
            1)))))))
debug       0
debug       ()
debug       1
debug       ()
debug       2
debug       (((1) 0))
debug       (0)
debug       3
debug       (((1) 0) ((0 1) 1))
debug       (0 0)
debug       4
debug       (((1) 0) ((0 1) 1) ((0 0 1) 2))
debug       (0 0 0)
debug       5
debug       (((1) 0) ((0 1) 1) ((0 0 1) 2) ((0 0 0 1) 3))
debug       (0 0 0 1)
debug       6
debug       (((1) 0) ((0 1) 1) ((0 0 1) 2) ((0 0 0 1) 3) ((0 0
             0 0 1) 4))
value       3


### ivmodel: Generates an instance of ivmodel
###          In this package, our IV model is
###          Y_i = beta_0 + D_i * beta_1 + X_i^T * gamma + epsilon_i
###          E(epsilon_i | Z_i, X_i) = 0
### INPUT: Y, outcome (n * 1 vector)
###        D, exposure (n * 1 vector)
###        Z, instruments (n * L matrix)
###        X, baseline and exogenous covariates (n * p matrix)
###        intercept, should we include the intercept term?
###        beta0: null value for beta
###        alpha: 1 - alpha confidence interval
###        k: values for the k-class estimator
###        heteroSE: heteroskedastic SE?
###        clusterSE: clustered SE?
###        clusterID, if clusterSE is TRUE, then one must provide a vector of
###                   length that's identical to the sample size.
###                   For example, if n = 6 and clusterID = c(1,1,1,2,2,2),
###                   there would be two clusters where the first cluster
###                   is formed by the first three observations and the
###                   second cluster is formed by the last three observations
###                   clusterID can be numeric, character, or factor.
###        deltarange: for sensitivity analysis.
### OUTPUT: ivmodel object which contains a clean-up version of Y,D,Z,X (if available)
ivmodel <- function(Y,D,Z,X,intercept=TRUE,
                    beta0=0,alpha=0.05,k=c(0,1),
                    manyweakSE = FALSE, heteroSE = FALSE, clusterID = NULL,
                    deltarange = NULL, na.action = na.omit) {

  # Error checking: check to see if necessary inputs are there!
  if(missing(Y)) stop("Y is missing!")
  if(missing(D)) stop("D is missing!")
  if(missing(Z)) stop("Z is missing!")

  # Error checking: check Y and D
  if( (!is.vector(Y) && !is.matrix(Y) && !is.data.frame(Y)) || (is.matrix(Y) && ncol(Y) != 1) || (is.data.frame(Y) && ncol(Y) != 1) || (!is.numeric(Y))) stop("Y is not a numeric vector.")
  if( (!is.vector(D) && !is.matrix(D) && !is.data.frame(Y)) || (is.matrix(D) && ncol(D) != 1) || (is.data.frame(D) && ncol(D) != 1) || (!is.numeric(D))) stop("D is not a numeric vector.")
  Y = as.numeric(Y); D = as.numeric(D)
  if(length(Y) != length(D)) stop("Dimension of Y and D are not the same!")

  # Error checking: check Z and convert "strings" into factors
  Z = data.frame(Z); stringIndex = sapply(Z,is.character); Z[stringIndex] = lapply(Z[stringIndex],as.factor)
  if(nrow(Z) != length(Y)) stop("Row dimension of Z and Y are not equal!")
  colnames(Z) = paste("Z",colnames(Z),sep="")

  # Add intercept as X
  if(intercept && !missing(X)) X = data.frame(X,1)
  if(intercept && missing(X)) X = data.frame(rep(1,length(Y)))

  # Error checking: check X and convert "strings" into factors
  if(!missing(X)) {
    X = data.frame(X); stringIndex = sapply(X,is.character); X[stringIndex] = lapply(X[stringIndex],as.factor)
	if(nrow(X) != length(Y)) stop("Row dimension of X and Y are not equal!")
	colnames(X) = paste("X",colnames(X),sep="")
  }

  # Coalesce all data into one data.frame
  if(!missing(X)) {
    allDataOrig = cbind(Y,D,Z,X)
  } else {
    allDataOrig = cbind(Y,D,Z)
  }

  # NA handling
  if(identical(na.action, na.fail) | identical(na.action, na.omit) | identical(na.action, na.pass)){
    allDataOrig = na.action(allDataOrig)
    naindex = rep(TRUE, nrow(allDataOrig))
	naindex[apply(is.na(allDataOrig), 1, any)] = NA
  }else if(identical(na.action, na.exclude)){
    naindex = rep(TRUE, nrow(allDataOrig))
	naindex[apply(is.na(allDataOrig), 1, any)] = NA
	allDataOrig = na.action(allDataOrig)
  }else{
    stop("Wrong input of NA handling!")
  }

  # Fit adjustment model
  ff = terms(Y ~ D + . -1,data=allDataOrig)
  mf = model.frame(ff,allDataOrig)

  # Declare Y and D
  Y = as.matrix(mf$Y); colnames(Y) = "Y"
  D = as.matrix(mf$D); colnames(D) = "D"

  # Extract Z and X
  allData = sparse.model.matrix(ff,allDataOrig); attr(allData,"assign") = NULL; attr(allData,"contrasts") = NULL
  Zindex = grep(paste("^",colnames(Z),sep="",collapse="|"),colnames(allData))
  Z = allData[,Zindex,drop=FALSE]; colnames(Z) = sub("^Z","",colnames(Z))
  if(!missing(X)) {
    Xindex = grep(paste("^",colnames(X),sep="",collapse="|"),colnames(allData)); X = allData[,Xindex,drop=FALSE]; colnames(X) = sub("^X","",colnames(X))
  }
  # Check to see if there's enough data points after removing missing values
  if(nrow(Y) <= 1) stop("Too much missing data!")

  n = length(Y)
  if(!missing(X)){
    ### clean X and project Y, D, Z
    qrX<-qr(X)
    p = qrrank(qrX)
    if(p==0)
      stop("vector in X are all 0")
    #if(p<ncol(X))
      #X<-qr.Q(qrX)[, 1:p]%*%qrRM(qrX)[1:p, 1:p]
    Yadj = as.matrix(qr.resid(qrX,Y)); Dadj = as.matrix(qr.resid(qrX,D)); Zadj = as.matrix(qr.resid(qrX,Z))
    ### clean Zadj
    ZadjQR = qr(Zadj)
    L = qrrank(ZadjQR)
    if(L==0)
      stop("No useful instrumental variables")
    if(L==1) {
      Zadj = qr.Q(ZadjQR)[, 1:L] * as.numeric(qrRM(ZadjQR)[1:L,1:L])
    }
    if(L<ncol(Z) && L> 1){
      Zadj<-qr.Q(ZadjQR)[, 1:L]%*%qrRM(ZadjQR)[1:L, 1:L]  ### shall we update the Z by doing qr(X, Z)?
      #qrXZ<-qr(cbind(X, Z))
      #Z<-(qr.Q(qrXZ)[, 1:(p+L)]%*%qrRM(qrXZ)[1:(p+L), 1:(p+L)])[,(p+1):(p+L)]
    }
    ivmodelObject = list(call = match.call(),n=n,L=L,p=p,Y=Y,D=D,Z=Z,X=X,Yadj=Yadj,Dadj=Dadj,Zadj=Zadj, ZadjQR = ZadjQR)

  }else{
    p = 0
    Yadj = Y; Dadj = D; Zadj = as.matrix(Z)
    ### only need to clean Z
    ZadjQR = qr(Zadj)
    L = qrrank(ZadjQR)
    if(L==0)
      stop("No useful instrumental variables")
    if(L == 1) {
      Z <- Zadj <-qr.Q(ZadjQR)[, 1:L] * as.numeric(qrRM(ZadjQR)[1:L, 1:L])
    }
    if(L<ncol(Z) && L > 1)
      Z<-Zadj<-qr.Q(ZadjQR)[, 1:L]%*%qrRM(ZadjQR)[1:L, 1:L]

    ivmodelObject = list(call = match.call(),n=n,L=L,p=p,Y=Y,D=D,Z=Z,X=NA,Yadj=Yadj,Dadj=Dadj,Zadj=Zadj, ZadjQR = ZadjQR)
  }

  class(ivmodelObject) = "ivmodel"

  ivmodelObject$naindex = naindex
  ivmodelObject$alpha = alpha
  ivmodelObject$beta0 = beta0
  ivmodelObject$deltarange = deltarange

  ivmodelObject$AR = AR.test(ivmodelObject,beta0=beta0,alpha=alpha)
  ivmodelObject$ARsens = ARsens.test(ivmodelObject,beta0=beta0,alpha=alpha,deltarange=deltarange)
  ivmodelObject$kClass = KClass(ivmodelObject,beta0=beta0,alpha=alpha,k=k,heteroSE=heteroSE,clusterID=clusterID) ## Don't use many weak IV asymptotics for OLS and TSLS
  ivmodelObject$LIML = LIML(ivmodelObject,beta0=beta0,alpha=alpha,manyweakSE=manyweakSE,heteroSE=heteroSE,clusterID=clusterID)
  ivmodelObject$Fuller = Fuller(ivmodelObject,beta0=beta0,alpha=alpha,manyweakSE=manyweakSE,heteroSE=heteroSE,clusterID=clusterID)
  ivmodelObject$CLR = CLR(ivmodelObject,beta0=beta0,alpha=alpha)

  return(ivmodelObject)
}


model_preciprec <- function (Sdry_t1 = 0.0,
         Sdry = 0.0,
         Swet = 0.0,
         Swet_t1 = 0.0,
         Sdepth_t1 = 0.0,
         Sdepth = 0.0,
         Mrf = 0.0,
         precip = 0.0,
         Snowaccu = 0.0,
         rho = 100.0){
    #'- Name: Preciprec -Version: 1.0, -Time step: 1
    #'- Description:
    #'            * Title: Precipitation ReCalculation
    #'            * Author: STICS
    #'            * Reference: doi:http://dx.doi.org/10.1016/j.agrformet.2014.05.002
    #'            * Institution: INRA
    #'            * Abstract: recalculation of precipitation
    #'- inputs:
    #'            * name: Sdry_t1
    #'                          ** description : water in solid state in the snow cover in previous day
    #'                          ** inputtype : variable
    #'                          ** variablecategory : state
    #'                          ** datatype : DOUBLE
    #'                          ** default : 0.0
    #'                          ** min : 0.0
    #'                          ** max : 500.0
    #'                          ** unit : mmW
    #'                          ** uri :
    #'            * name: Sdry
    #'                          ** description : water in solid state in the snow cover
    #'                          ** inputtype : variable
    #'                          ** variablecategory : state
    #'                          ** datatype : DOUBLE
    #'                          ** default : 0.0
    #'                          ** min : 0.0
    #'                          ** max : 500.0
    #'                          ** unit : mmW
    #'                          ** uri :
    #'            * name: Swet
    #'                          ** description : water in liquid state in the snow cover
    #'                          ** inputtype : variable
    #'                          ** variablecategory : state
    #'                          ** datatype : DOUBLE
    #'                          ** default : 0.0
    #'                          ** min : 0.0
    #'                          ** max : 100.0
    #'                          ** unit : mmW
    #'                          ** uri :
    #'            * name: Swet_t1
    #'                          ** description : water in liquid state in the snow cover in previous day
    #'                          ** inputtype : variable
    #'                          ** variablecategory : state
    #'                          ** datatype : DOUBLE
    #'                          ** default : 0.0
    #'                          ** min : 0.0
    #'                          ** max : 100.0
    #'                          ** unit : mmW
    #'                          ** uri :
    #'            * name: Sdepth_t1
    #'                          ** description : snow cover depth Calculation in previous day
    #'                          ** inputtype : variable
    #'                          ** variablecategory : state
    #'                          ** datatype : DOUBLE
    #'                          ** default : 0.0
    #'                          ** min : 0.0
    #'                          ** max : 5000.0
    #'                          ** unit : m
    #'                          ** uri :
    #'            * name: Sdepth
    #'                          ** description : snow cover depth Calculation
    #'                          ** inputtype : variable
    #'                          ** variablecategory : state
    #'                          ** datatype : DOUBLE
    #'                          ** default : 0.0
    #'                          ** min : 0.0
    #'                          ** max : 5000.0
    #'                          ** unit : m
    #'                          ** uri :
    #'            * name: Mrf
    #'                          ** description : liquid water in the snow cover in the process of refreezing
    #'                          ** inputtype : variable
    #'                          ** variablecategory : rate
    #'                          ** datatype : DOUBLE
    #'                          ** default : 0.0
    #'                          ** min :
    #'                          ** max :
    #'                          ** unit : mmW/d
    #'                          ** uri :
    #'            * name: precip
    #'                          ** description : current precipitation
    #'                          ** inputtype : variable
    #'                          ** variablecategory : auxiliary
    #'                          ** datatype : DOUBLE
    #'                          ** default : 0.0
    #'                          ** min : 0.0
    #'                          ** max : 5000.0
    #'                          ** unit : mmW
    #'                          ** uri :
    #'            * name: Snowaccu
    #'                          ** description : snowfall accumulation
    #'                          ** inputtype : variable
    #'                          ** variablecategory : rate
    #'                          ** datatype : DOUBLE
    #'                          ** default : 0.0
    #'                          ** min :
    #'                          ** max :
    #'                          ** unit : mmW/d
    #'                          ** uri :
    #'            * name: rho
    #'                          ** description : The density of the new snow fixed by the user
    #'                          ** inputtype : parameter
    #'                          ** parametercategory : constant
    #'                          ** datatype : DOUBLE
    #'                          ** default : 100
    #'                          ** min :
    #'                          ** max :
    #'                          ** unit : kg/m**3
    #'                          ** uri :
    #'- outputs:
    #'            * name: preciprec
    #'                          ** description : recalculated precipitation
    #'                          ** variablecategory : state
    #'                          ** datatype : DOUBLE
    #'                          ** min : 0.0
    #'                          ** max : 500.0
    #'                          ** unit : mmW
    #'                          ** uri :
    preciprec <- precip
    if (Sdry + Swet < Sdry_t1 + Swet_t1)
    {
        preciprec <- preciprec + ((Sdepth_t1 - Sdepth) * rho) - Mrf
    }
    preciprec <- preciprec - Snowaccu
    return (list('preciprec' = preciprec))
}