# Benjamini-Hochberg procedure

bh_apply <- function(data, alpha=0.05, time_windows=list(c(300, 500), c(600, 1000))) {
        tws <- time_windows
        dt <- data[, lapply(.SD, mean), by=list(Timestamp, Electrode), .SDcols=colnames(data)[grep("pval", colnames(data))]]
        for (i in colnames(dt)[grep("pval", colnames(dt))]) {
                dt[, paste0("sig_", i)] <- rep(FALSE, nrow(dt))
                for (j in tws) {
                        signi <- (p.adjust(dt[Timestamp >= j[1] & Timestamp <= j[2], ..i][[1]], method='fdr') < alpha)
                        indi <- which(dt$Timestamp >= j[1] & dt$Timestamp <= j[2])
                        dt[indi, paste0("sig_", i) := signi]
                }
        }

        cols <- c("Timestamp", "Electrode", colnames(dt)[grep("sig", colnames(dt))])
        merge(data, dt[,..cols], by=c("Timestamp", "Electrode"))
}

bh_apply_bonf <- function(data, alpha=0.05, time_windows=list(c(300, 500), c(600, 1000))) {
        tws <- time_windows
        dt <- data[, lapply(.SD, mean), by=list(Timestamp, Electrode), .SDcols=colnames(data)[grep("pval", colnames(data))]]
        for (i in colnames(dt)[grep("pval", colnames(dt))]) {
                dt[, paste0("sig_", i)] <- rep(FALSE, nrow(dt))
                for (j in tws) {
                        signi <- (p.adjust(dt[Timestamp >= j[1] & Timestamp <= j[2], ..i][[1]], method='none') < alpha)
                        indi <- which(dt$Timestamp >= j[1] & dt$Timestamp <= j[2])
                        dt[indi, paste0("sig_", i) := signi]
                }
        }

        cols <- c("Timestamp", "Electrode", colnames(dt)[grep("sig", colnames(dt))])
        merge(data, dt[,..cols], by=c("Timestamp", "Electrode"))
}

bh_apply_sepelec <- function(data, alpha=0.05, time_windows=list(c(300, 500), c(600, 1000))) {
        tws <- time_windows
        elecs <- c("Fz", "Cz", "Pz")
        dt <- data[, lapply(.SD, mean), by=list(Timestamp, Electrode), .SDcols=colnames(data)[grep("pval", colnames(data))]]
        for (i in colnames(dt)[grep("pval", colnames(dt))]) {
                dt[, paste0("sig_", i)] <- rep(FALSE, nrow(dt))
                for (j in tws) {
                        for (e in elecs) {
                                signi <- (p.adjust(dt[Timestamp >= j[1] & Timestamp <= j[2] & Electrode == e, ..i][[1]], method='fdr') < alpha)
                                indi <- which(dt$Timestamp >= j[1] & dt$Timestamp <= j[2] & dt$Electrode == e)
                                dt[indi, paste0("sig_", i) := signi]
                        }
                }
        }

        cols <- c("Timestamp", "Electrode", colnames(dt)[grep("sig", colnames(dt))])
        merge(data, dt[,..cols], by=c("Timestamp", "Electrode"))
}

bh_apply_poolall <- function(data, alpha=0.05, time_windows=list(c(300, 500), c(600, 1000))) {
        tws <- time_windows
        elecs <- c("Fz", "Cz", "Pz")
        dt <- data[, lapply(.SD, mean), by=list(Timestamp, Electrode), .SDcols=colnames(data)[grep("pval", colnames(data))]]
        for (i in colnames(dt)[grep("pval", colnames(dt))]) {
                dt[, paste0("sig_", i)] <- rep(FALSE, nrow(dt))
                        signi <- (p.adjust(dt[(Timestamp >= 300 & Timestamp <= 500) | (Timestamp >= 600 & Timestamp <= 1000), ..i][[1]], method='fdr') < alpha)
                        indi <- which((dt$Timestamp >= 300 & dt$Timestamp <= 500) | (dt$Timestamp >= 600 & dt$Timestamp <= 1000))
                        dt[indi, paste0("sig_", i) := signi]
        }

        cols <- c("Timestamp", "Electrode", colnames(dt)[grep("sig", colnames(dt))])
        merge(data, dt[,..cols], by=c("Timestamp", "Electrode"))
}

bh_none <- function(data, alpha=0.05, time_windows=list(c(300, 500), c(600, 1000))) {
        tws <- time_windows
        elecs <- c("Fz", "Cz", "Pz")
        dt <- data[, lapply(.SD, mean), by=list(Timestamp, Electrode), .SDcols=colnames(data)[grep("pval", colnames(data))]]
        for (i in colnames(dt)[grep("pval", colnames(dt))]) {
                dt[, paste0("sig_", i)] <- rep(FALSE, nrow(dt))
                signi <- dt[(dt$Timestamp >= 300 & dt$Timestamp <= 500) | (dt$Timestamp >= 600 & dt$Timestamp <= 1000),..i] < alpha
                indi <- which((dt$Timestamp >= 300 & dt$Timestamp <= 500) | (dt$Timestamp >= 600 & dt$Timestamp <= 1000))
                dt[indi, paste0("sig_", i) := signi]
        }

        cols <- c("Timestamp", "Electrode", colnames(dt)[grep("sig", colnames(dt))])
        merge(data, dt[,..cols], by=c("Timestamp", "Electrode"))
}

bh_identify_cutoff <- function(pvals, alpha = 0.05)
{
        pvals <- sort(pvals)     # sort ascending
        m     <- length(pvals)   # number of hypotheses
        k     <- 1               # k iterator
        for (k in seq(1:m)){
                if (pvals[k] >= k/m * alpha) {
                        break()
                }
        }
        # while (pvals[k] < (k / m) * alpha)
        #         k <- k + 1
        return(pvals[k])
}

bh_plot <- function(pvals, alpha = 0.05)
{
        pvals <- sort(pvals)    # sort ascending
        m     <- length(pvals)  # number of hypothesis
        ks    <- seq(1, length(pvals), 1)
        plot(pvals ~ ks)
        abline(0, alpha / m)
}


################################################################################
#
# Simulation script
#
# Andrew Robinson 15-April-2015
#
################################################################################

the.seed <- 10000000

reps <- 1000
cores <- 25

MAIL <- TRUE

mail <- function(address, subject, message, attach = NULL) {
  if (MAIL)
    if (is.null(attach)) {
      system(paste("echo '", message,
                   "' | mutt -s '", subject,
                   "' ", address, sep=""))
    } else {
      system(paste("echo '", message,
                   "' | mutt -s '", subject, "'",
                   " ", address,
                   paste(" -a ", attach, collapse=" "), sep=""))
    }
}

mail.and.stop <- function() {
  mail("mensurationist@gmail.com",
       "Sims on server: ERROR.",
       "Stopped.")
  stop()
}

options(error = mail.and.stop)

library(lme4)    # For mixed-effects models

library(Amelia)  # For imputation

library(equivalence) # For data that I know

library(parallel) # Speed things up

data(ufc)

str(ufc)

ufc <- subset(ufc, !is.na(Dbh))
ufc$Species[ufc$Species %in% c("F","FG")] <- "GF"
ufc$Species <- factor(ufc$Species)
ufc$ht.measured <- is.na(ufc$Height)

ufc <- subset(ufc,
              Species %in% names(sort(-table(Species)))[1:6],
              select = c("ht.measured","Plot","Dbh.in","Species"))

## Fit a pair of models to compare using LRT

ufc.whole <- ufc

isna.1.w <- glmer(ht.measured ~ Dbh.in + Species + (1 | Plot),
                  data = ufc.whole, family = binomial)
isna.0.w <- glmer(ht.measured ~ Dbh.in + (1 | Plot),
                  data = ufc.whole, family = binomial)

anova(isna.0.w, isna.1.w)

################################################################################

imputeLRT <- function(h0, h1, imputed.data.list) {
  m <- length(imputed.data.list)
  h0.models <- lapply(imputed.data.list,
                      function (dataset) {
                        update(h0, data = dataset)
                      })
  h0.dev.funs <- lapply(imputed.data.list,
                        function (dataset) {
                          update(h0, data = dataset,
                                 devFunOnly = TRUE)
                        })
  Q.bar.0 <-
    colMeans(do.call(rbind,
                     lapply(h0.models,
                            function (fit) {
                              c(getME(fit, "theta"), fixef(fit))
                            })))
  h1.models <- lapply(imputed.data.list,
                      function (dataset) {
                        update(h1, data = dataset)
                      })
  h1.dev.funs <- lapply(imputed.data.list,
                        function (dataset) {
                          update(h1, data = dataset,
                                 devFunOnly = TRUE)
                        })
  Q.bar.1 <-
    colMeans(do.call(rbind,
                     lapply(h1.models,
                            function (fit) {
                              c(getME(fit, "theta"), fixef(fit))
                            })))
  d.prime.m.bar <- mean(unlist(lapply(1:m,
                                      function(i) {
                                        anova(h0.models[[i]],
                                              h1.models[[i]])$Chisq[2]
                                      })))
  d.L.bar <- mean(unlist(lapply(1:m,
                                function(i) {
                                  h0.dev.funs[[i]](Q.bar.0) -
                                    h1.dev.funs[[i]](Q.bar.1)
                                })))
  p0 <- length(Q.bar.0)
  p1 <- length(Q.bar.1)
  k <- p1 - p0
  rL <- (m + 1) / ((m - 1) * k) * (d.prime.m.bar - d.L.bar) # 3.8
  D.L <- d.L.bar / ((1 + rL) * k)                           # 3.7
  v <- k * (m - 1)
  w <- ifelse(v > 4,                                        # 2.7
              4 + (v - 4) * (1 + (1 - v/2) / rL)^2,
              v / 2 * (1 + 1/k) * (1 + 1/rL)^2)
  Pval <- 1 - pf(D.L, k, w)
  return(list(
    D.L = D.L,
    Pval = Pval,
    rL = rL,
    d.L.bar = d.L.bar,
    d.prime.m.bar = d.prime.m.bar,
    Q.bar.0 = Q.bar.0,
    Q.bar.1 = Q.bar.1,
    k = k,
    w = w,
    p0 = p0,
    p1 = p1,
    m = m))
}

isna.1 <- glmer(ht.measured ~ Dbh.in + Species + (1 | Plot),
                data = ufc, family = binomial)
isna.0 <- glmer(ht.measured ~ Dbh.in + (1 | Plot),
                data = ufc, family = binomial)

imputeLRT(isna.0, isna.1,
          list(ufc, ufc, ufc, ufc, ufc))

anova(isna.0, isna.1)

simulate <- function(m = 5, n = 30) {
  make.missing <- sample(1:nrow(ufc), size = n * 3, replace = FALSE)
  is.na(ufc$Species[make.missing[1:n]]) <- TRUE
  is.na(ufc$Dbh.in[make.missing[(n+1):(2*n)]]) <- TRUE
  is.na(ufc$ht.measured[make.missing[(2*n+1):(3*n)]]) <- TRUE
  isna.1 <- glmer(ht.measured ~ Dbh.in + Species + (1 | Plot),
                  data = ufc, family = binomial)
  isna.0 <- glmer(ht.measured ~ Dbh.in + (1 | Plot),
                  data = ufc, family = binomial)
  naive.p <- anova(isna.0, isna.1)[["Pr(>Chisq)"]][2]
  ## Impute datasets using Amelia
  ufc.imputes <- amelia(ufc,
                        m = m,
                        cs = "Plot",
                        noms = "Species",
                        p2s = 0)
  imputed <- imputeLRT(isna.0, isna.1, ufc.imputes$imputations)
  return(list(naive.p = naive.p, imputed.p = imputed$Pval))
}

simulate()

frame <- expand.grid(ms = c(2, 5, 10, 20, 50, 100),
                     ns = c(5, 10, 20, 30, 50, 100),
                     reps = 1:reps)

results <- mclapply(1:nrow(frame),
                    function(i)
                    simulate(frame$ms[i], frame$ns[i]),
                    mc.cores = cores)

results <- do.call(rbind, results)

experiment <- cbind(frame, results)

save(experiment, file = "mi.RData")

mail(address = "andrewpr",
     subject = "MI simulations",
     message = "All done.",
     attach = "mi.RData")


subroutine locn(i,j,kj,nadj,madj,x,y,ntot,eps)

# Find the appropriate location for j in the adjacency list
# of i.  This is the index which j ***will*** have when
# it is inserted into the adjacency list of i in the
# appropriate place.  Called by insrt.

implicit double precision(a-h,o-z)
dimension nadj(-3:ntot,0:madj), x(-3:ntot), y(-3:ntot)
logical before

n = nadj(i,0)

# If there is nothing already adjacent to i, then j will have place 1.
if(n==0) {
        kj = 1
        return
}

# Run through i's list, checking if j should come before each element
# of that list.  (I.e. if i, j, and k are in anti-clockwise order.)
# If j comes before the kj-th item, but not before the (kj-1)-st, then
# j should have place kj.
do ks = 1,n {
	kj = ks
        k = nadj(i,kj)
        call acchk(i,j,k,before,x,y,ntot,eps)
        if(before) {
                km = kj-1
                if(km==0) km = n
                k = nadj(i,km)
                call acchk(i,j,k,before,x,y,ntot,eps)
                if(before) next
                # If j is before 1 and after n, then it should
                # have place n+1.
                if(kj==1) kj = n+1
                return
        }
}

# We've gone right through the list and haven't been before
# the kj-th item ***and*** after the (kj-1)-st on any occasion.
# Therefore j is before everything (==> place 1) or after
# everything (==> place n+1).
if(before) kj = 1
else kj = n+1

return
end


subroutine swap(j,k1,k2,shdswp,nadj,madj,x,y,ntot,eps,nerror)

# The segment k1->k2 is a diagonal of a quadrilateral
# with a vertex at j (the point being added to the
# triangulation).  If the LOP is not satisfied, swap
# it for the other diagonal.
# Called by addpt.

implicit double precision(a-h,o-z)
dimension nadj(-3:ntot,0:madj), x(-3:ntot), y(-3:ntot)
dimension ntadj(1000)
logical shdswp, anticl

# If vertices k1 and k2 are not connected there is no diagonal to swap.
# This could happen if vertices j, k1, and k2 were colinear, but shouldn't.
call adjchk(k1,k2,shdswp,nadj,madj,ntot,nerror)
if(nerror > 0) {
    return
}
if(!shdswp) return

# Get the other vertex of the quadrilateral.
call pred(k,k1,k2,nadj,madj,ntot,nerror) # If these aren't the same, then
if(nerror > 0) return
call succ(kk,k2,k1,nadj,madj,ntot,nerror) # there is no other vertex.
if(nerror > 0) return
if(kk!=k) {
#    if(j==580) call intpr("no other vertex",-1,1,0)
        shdswp = .false.
        return
}

# Check whether the LOP is satisified; i.e. whether
# vertex k is outside the circumcircle of vertices j, k1, and k2
if(k==580) {
    call intpr("From swap; point being added =",-1,j,1)
# Adj. list of k1 ("now").
    nk1 = nadj(k1,0)
    do jc = 1,nk1 {
            ntadj(jc) = nadj(k1,jc)
    }
    call intpr("now =",-1,k1,1)
    call intpr("adjacency list of now:",-1,ntadj,nk1)
# Adj. list of k1 ("now").
    nk2 = nadj(k2,0)
    do jc = 1,nk2 {
            ntadj(jc) = nadj(k2,jc)
    }
    call intpr("nxt =",-1,k2,1)
    call intpr("adjacency list of nxt:",-1,ntadj,nk2)
# Adj. list of j ("point being added").
    nj = nadj(j,0)
    do jc = 1,nj {
            ntadj(jc) = nadj(j,jc)
    }
    call intpr("point being added =",-1,j,1)
    call intpr("adjacency list of point being added:",-1,ntadj,nj)
# j, now, nxt should be in anticlockwise order.
    call acchk(j,k1,k2,anticl,x,y,ntot,eps)
    if(anticl) {
        call intpr("anticlockwise",-1,1,0)
    } else {
        call intpr("clockwise",-1,1,0)
    }
#
# i = now = k1, k = nxt = k2, and j = other vertex = k:
}
call qtest(j,k1,k,k2,shdswp,x,y,ntot,eps,nerror)
if(nerror > 0) return

# Do the actual swapping.
if(shdswp) {
        call delet(k1,k2,nadj,madj,ntot,nerror)
	if(nerror > 0) return
	call insrt(j,k,nadj,madj,x,y,ntot,nerror,eps)
	if(nerror > 0) return
}
return
end


\docType{data}
\name{paises}
\alias{paises}
\title{Datos de Gapminder}
\format{El cuadro de datos principal tiene 1704 filas y 6 columnas
\describe{
\item{pais}{países incluídos (factor con 142 niveles)}
\item{continente}{continentes (factor con 5 niveles)}
\item{anio}{desde 1952 a 2007, datos cada 5 años}
\item{esperanza_de_vida}{esperanza de vida al nacer, en años}
\item{poblacion}{población}
\item{pib_per_capita}{PIB per cápita (en dólares americanos, ajustados según inflación)}
}}
\usage{paises}
\description{Extracto de datos de Gapminder sobre expectativa de vida, PIB per cápita y población, según país}
\source{http://www.gapminder.org/data/}
\keyword{datasets}


# 3. faza: Vizualizacija podatkov

# Uvozimo zemljevid.
zemljevid <- uvozi.zemljevid("http://baza.fmf.uni-lj.si/OB.zip", "OB",
                             pot.zemljevida="OB", encoding="Windows-1250")
levels(zemljevid$OB_UIME) <- levels(zemljevid$OB_UIME) %>%
  { gsub("Slovenskih", "Slov.", .) } %>% { gsub("-", " - ", .) }
zemljevid$OB_UIME <- factor(zemljevid$OB_UIME, levels=levels(obcine$obcina))
zemljevid <- fortify(zemljevid)

# Izračunamo povprečno velikost družine
povprecja <- druzine %>% group_by(obcina) %>%
  summarise(povprecje=sum(velikost.druzine * stevilo.druzin) / sum(stevilo.druzin))

stevilo.po.letih <- data %>% group_by(leto) %>% summarise(Stevilo=sum(stevilo, na.rm = TRUE))

#Graf po stevilu priseljenih po letih
graf1 <- ggplot(stevilo.po.letih) + aes(x = leto, y = Stevilo) +
  geom_bar(stat = 'identity',position = 'dodge', fill = "darkcyan") +
  xlab("Leto") + ylab("Število") + ggtitle("Število priseljenih glede na leto")
graf1

#Graf po stevilu priseljenih po namenu
stevilo.po.namenu <- data %>% group_by(namen) %>% summarise(Stevilo=sum(stevilo, na.rm = TRUE))
graf2 <- ggplot(stevilo.po.namenu) + aes(x = namen, y = Stevilo) +
  geom_bar(stat = 'identity',position = 'dodge', fill = "darksalmon") +
  xlab("Leto") + ylab("Namen") + ggtitle("Število priseljenih glede na namen")
#2021/22
###################################################################################

#mojzemljevid

library(tmap)

zemljevid <- uvozi.zemljevid("http://biogeo.ucdavis.edu/data/gadm2.8/shp/SVN_adm_shp.zip","SVN_adm1", encoding = "UTF-8")
zemljevid$NAME_1 <- c("Gorenjska", "Goriška","Jugovzhodna Slovenija", "Koroška", "Primorsko-notranjska", "Obalno-kraška",
                      "Osrednjeslovenska", "Podravska", "Pomurska", "Savinjska", "Posavska", "Zasavska")

# 1.zemljevid: priseljevanje prebivalstva po regijah
regije_pri <- medregijske %>% group_by(regijapri) %>% summarise(skupaj = sum(stevilo, na.rm = TRUE ))
regije_pri$regijapri = regije_pri$regijapri %>% trimws()

podatki_pri = merge(zemljevid, regije_pri, by.x = "NAME_1", by.y = "regijapri" )
tm_shape(podatki_pri) +
  tm_polygons("skupaj") +
  tm_format("NLD", title="Število prebivalstva, ki se je priselilo iz določene regije", bg.color="white")

#2. zemljevid: izseljevanje prebivalstva po regijah
regije_izs <- medregijske %>% group_by(regijaiz) %>% summarise(skupaj = sum(stevilo, na.rm = TRUE ))
regije_izs$regijaiz = regije_izs$regijaiz %>% trimws()

podatki_izs = merge(zemljevid, regije_izs, by.x = "NAME_1", by.y = "regijaiz" )
tm_shape(podatki_izs) +
  tm_polygons("skupaj") +
  tm_format("NLD", title="Število prebivalstva, ki se je izselilo iz določene regije", bg.color="white")

#ŠE LEGENDO UREDIT!

#Graf 1: Povprečno število prebivalstva, ki se je selilo glede na statistične regije Slovenije

graf1 <- ggplot(data = povprecje_regije, aes(x=" " ,y=povprecje, fill = regijaiz)) +
  geom_bar(stat="identity", position = 'dodge') +
  facet_wrap(~regijapri, ncol= 6) +
  xlab(" ") + ylab("Povprečno število") +
  ggtitle("Priseljevanje in odseljevanje po statističnih regijah Slovenije") +
  theme(plot.title = element_text(family="Trebuchet MS", face="bold", size=20, hjust=0, color="black")) +
  theme(axis.text.x = element_text(angle=90))
+ coord_flip() + theme_dark() +
  scale_fill_brewer(palette = "BrBG")

#Graf 2: Število priseljenih in izseljenih ljudi glede na drzavo selitve

graf2 <- ggplot(data=povprecje2, aes(x=drzava, y=povprecje, fill=`vrsta`)) +
  geom_col(position = 'dodge') +
  coord_flip() +
  labs(x = "Država", y = "Povprečno število", title = "Povprečno število v priseljenega ali odseljenega prebivalstva glede na države") +
  theme_dark() +
  scale_fill_brewer(palette = "BrBG")

odseljeni<- meddrzavne %>% filter( vrsta != "Priseljeni iz tujine") %>% select(- 'vrsta') %>%
  group_by (drzava, spol) %>% summarise(Stevio=sum(stevilo))

priseljeni <- meddrzavne %>% filter( vrsta != "Odseljeni iz tujino") %>% select(- 'vrsta') %>%
  group_by (drzava, spol) %>% summarise(Stevilo=sum(stevilo)) #to bos potrebovala za pielibr

# Graf 3: Graf, ki prikazuje število priseljenega prebivalstva glede na namen selitve in državo predhodnega bivališča.

graf3b <- ggplot(namen_priseljevanja, aes(x=leto,y=stevilo, group=1)) +
  geom_point(aes(col=drzava, size=stevilo)) +
  facet_grid(namen ~ .) + xlab("Leto") + ylab("Število") +
  ggtitle("Priseljeni prebivalci glede na namen selitve po letih") +
  coord_flip() + theme_dark() +
  scale_fill_brewer(palette = "BrBG")

#Graf 4: Graf, ki prikazuje število priseljenega prebivalstva glede na njihovo izobrazvo in državo predhodnega bivališča

povprecje4 <- izobrazba_priseljeni %>% group_by(drzava, izobrazba) %>%
  summarise(povprecje=(sum(stevilo)/9))

graf4 <- ggplot(izobrazba_priseljeni, aes(x=leto,y=stevilo, group=1)) +
  geom_point(aes(col=drzava, size=stevilo)) +
  facet_grid(izobrazba ~ .) +
  labs(y="Število ljudi",
      x="Vrsta izobrazbe",
      title="Število priseljenega prebivalstva glede na stopnjo izobrazbe") +
  theme_dark() +
  scale_fill_brewer(palette = "BrBG")

#Graf 5: Graf, ki prikazuje število izseljenega prebivalstva glede na njihovo izobrazbo in starostno skupino

#starost <- c("15-24", "25-54", "55-64") zdruziii

povprecje5 <- izobrazba_izseljeni %>% group_by(starost, izobrazba) %>%
  summarise(povprecje=(sum(stevilo)/9))

graf5 <- ggplot(povprecje5, aes(x=izobrazba, y=povprecje)) +
          geom_point(aes(col=starost, size=povprecje)) +
          geom_smooth(method="loess", se=F) +
          labs(y="Število ljudi",
          x="Vrsta izobrazbe",
          title="Povprečno število izseljenega prebivalstva glede na starostno skupino in vrsto izobrazbe") +
          theme_dark() +
          scale_fill_brewer(palette = "BrBG")

graf5a <- ggplot(data=izobrazba_izseljeni, aes(x=leto, y=stevilo, col=spol)) +
  geom_line(size=1) + facet_grid(~starost) +
  theme_bw() +
  scale_color_manual(values=c("darkgoldenrod3", "cadetblue")) +
  xlab("Država") + ylab("Povprečno število(/100)") +
  ggtitle("Odseljeni prebivalci glede na status aktivnosti") +
  coord_flip() +
  theme_dark() +
  scale_fill_brewer(palette = "BrBG") +
  scale_x_continuous(breaks=seq(2010, 2020, 4))

#Graf 6: Graf, ki prikazuje število izeljenega prebivalstva glede na status aktivosti in državo pihodnjega bivališča
povprecje6 <- izseljeni_status_akt %>% group_by(drzava, status) %>%
  summarise(povprecje=(sum(stevilo)/9)/100)

graf6 <- ggplot(data=povprecje6, aes(x = drzava, y = povprecje, fill = status)) +
  geom_bar(stat = 'identity', position = 'dodge') +
  xlab("Država") + ylab("Povprečno število(/100)") +
  ggtitle("Odseljeni prebivalci glede na status aktivnosti") +
  coord_flip() +
  theme_dark() +
  scale_fill_brewer(palette = "BrBG")
 #dva grafa: celine pa evropa po drzavah

#####################################################################


#!/usr/bin/env Rscript
#
# Randomly draw distributions of fitness effects for variants and use those
# to simulate S&R evolution.
#
# Can either specify "all" to draw fitness effects from an exponential
# distribution, or a number to specify the number of SNPs that will have
# an effect (all others set to zero).
#

library(boot)
library(data.table)
library(dplyr)
library(dtplyr)

## Program options:
cargs = commandArgs(trailingOnly=TRUE)
genotype_file = cargs[1]
n_snps = cargs[2]
n_generations = as.numeric(cargs[3])
min_maf = as.numeric(cargs[4])
max_miss = as.numeric(cargs[5])
output_prefix = cargs[6]

## Probability that the reference allele is more fit than the alternate
ref_allele_more_fit = 0.8
if(length(cargs) > 6) {
    ref_allele_more_fit = as.numeric(cargs[7])
}

## Read in data
genotypes = fread(genotype_file, header=TRUE) %>%
    select(-pos, -contig, -rs) %>%
    as.matrix()

genotype_names = fread(genotype_file, header=TRUE) %>%
    select(pos, contig, rs)

initial_allele_freqs = apply(genotypes, 1, function(x) {
                             ref = sum(!(is.na(x)) & x == 1)
                             alt = sum(!(is.na(x)) & x == 0)
                             ref / (ref + alt)
})

## Filter data
maf = ifelse(initial_allele_freqs < 0.5, initial_allele_freqs,
             1 - initial_allele_freqs)
missingness = apply(genotypes, 1, function(x) sum(is.na(x)) / length(x))

keep = maf > min_maf & missingness < max_miss
genotypes = genotypes[keep, ]
genotype_names = genotype_names[keep, ]
initial_allele_freqs = initial_allele_freqs[keep]
maf = maf[keep]
missingness = missingness[keep]

## Draw fitness effects
if(!is.na(as.numeric(n_snps))) {
    ## Set a few SNPs to fitness effect of 1, all others = 0
    ns = as.numeric(n_snps)
    x = rep(0, nrow(genotypes))
    x[sample(1:nrow(genotypes), ns, FALSE)] = sample(c(1, -1), ns, TRUE,
                                                     prob=c(1-ref_allele_more_fit, ref_allele_more_fit))
    snp_fitness_effects = x
} else {
    ## Draw from exponential distribution for all SNPs
    snp_fitness_effects = rexp(nrow(genotypes)) * ifelse(sample(c(TRUE, FALSE), nrow(genotypes), TRUE,
                                                                prob=c(1-ref_allele_more_fit, ref_allele_more_fit)),
                                                         -1, 1)
}

## Simulate fitness of strains and calculate allele frequency and
## allele frequency change. Takes fitness effects as Wrightian (average
## number of offspring).

## 1) Column-by-column, multiply the fitness_effect of the alternate
##    allele by whether the alternate allele is present. Do this in
##    a way that doesn't give any negative numbers (if alternate allele
##    effect is negative, make it a positive effect of the ref. allele).
## 2) Row-by-row, set missing genotypes to have the average effect
## 3) Add up all the fitness effects in a strain and divide by the
##    mean across strains.
## Note that in the genotype input file 1 = reference, 0 = alternate
snp_effects =
    apply(genotypes, 2, function(x) {
          snp_fitness_effects * ifelse(snp_fitness_effects < 0, -x, 1-x)
         }) %>%
    apply(., 1, function(x) {
          mw = mean(x, na.rm=TRUE)
          ifelse(is.na(x), mw, x)
         }) %>%
    t()
fitnesses = colSums(snp_effects) / mean(colSums(snp_effects))

## Calculate final strain frequencies assuming equal starting frequencies.
## Uses Wrightian fitness (fitness = average number of offspring).
initial_strain_freqs = rep(1/length(fitnesses), length(fitnesses))
n_inds = (fitnesses^n_generations)
strain_freqs = n_inds / sum(n_inds)
stopifnot(all(strain_freqs >= 0))
relative_fitnesses = log2(strain_freqs / initial_strain_freqs)
## If fitness = 0, need to do a rough correction:
relative_fitnesses[is.na(relative_fitnesses) | is.infinite(relative_fitnesses)] =
    min(relative_fitnesses[!is.na(relative_fitnesses) & !(is.infinite(relative_fitnesses))]) - 2

## Calculate final allele frequencies
stopifnot(all(names(strain_freqs) == colnames(genotypes)))
allele_freqs = apply(genotypes, 1, function(x) {
                     ref = sum(strain_freqs[!(is.na(x)) & x == 1])
                     alt = sum(strain_freqs[!(is.na(x)) & x == 0])
                     ref / (ref + alt)
         })
stopifnot(all(allele_freqs >= 0 & allele_freqs <= 1))

## Calculate LD groups from final strain frequencies for the top 2000
## snps (by allele frequency change); note groups with ties may not
## be completely included in the top 5000. Order of ties is arbitrary.
n_ld = 5000
af_change = abs(allele_freqs - initial_allele_freqs)
af_order = order(order(af_change, decreasing=TRUE))
ld_groups_95 = numeric(length(af_change)) * NaN
grp = -1
for(i in 1:(n_ld-1)) {
    ii = which(af_order == i)
    if(!is.na(ld_groups_95[ii])) next
    grp = grp + 1
    ld_groups_95[ii] = grp
    for(j in (i+1):n_ld) {
        jj = which(af_order == j)
        if(!is.na(ld_groups_95[jj])) next
        g = cbind(genotypes[ii, ], genotypes[jj, ])
        r2 = (corr(g[!(is.na(g[, 1])) & !(is.na(g[, 2])), ],
                   w=strain_freqs[!(is.na(g[, 1])) & !(is.na(g[, 2]))]))^2
        if(is.na(r2)) {
           warning(paste('R^2 is NA at', genotype_names[['rs']][i],
                         genotype_names[['rs']][j]))
           next
         }
        if(r2 >= 0.95) {
            ld_groups_95[jj] = grp
        }
    }
}
i = n_ld
ii = which(af_order == i)
if(is.na(ld_groups_95[ii])) {
    grp = grp + 1
    ld_groups_95[ii] = grp
}

ld_groups_80 = numeric(length(af_change)) * NaN
grp = -1
for(i in 1:(n_ld-1)) {
    ii = which(af_order == i)
    if(!is.na(ld_groups_80[ii])) next
    grp = grp + 1
    ld_groups_80[ii] = grp
    for(j in (i+1):n_ld) {
        jj = which(af_order == j)
        if(!is.na(ld_groups_80[jj])) next
        g = cbind(genotypes[ii, ], genotypes[jj, ])
        r2 = (corr(g[!(is.na(g[, 1])) & !(is.na(g[, 2])), ],
                   w=strain_freqs[!(is.na(g[, 1])) & !(is.na(g[, 2]))]))^2
        if(is.na(r2)) {
           warning(paste('R^2 is NA at', genotype_names[['rs']][i],
                         genotype_names[['rs']][j]))
           next
         }
        if(r2 >= 0.80) {
            ld_groups_80[jj] = grp
        }
    }
}
i = n_ld
ii = which(af_order == i)
if(is.na(ld_groups_80[ii])) {
    grp = grp + 1
    ld_groups_80[ii] = grp
}

## Write output:

##  1. Phenotype file for Plink with fitness of strains
##     (for external association testing)
write.table(cbind(names(fitnesses), names(fitnesses), fitnesses),
            file=paste0(output_prefix, '.fitness.txt'),
            sep='\t', col.names=FALSE, row.names=FALSE, quote=FALSE)

##  2. Phenotype file with relative fitness
write.table(cbind(names(relative_fitnesses), names(relative_fitnesses), relative_fitnesses),
            file=paste0(output_prefix, '.relative_fitness.txt'),
            sep='\t', col.names=FALSE, row.names=FALSE, quote=FALSE)

##  3. Table of fitness effects and allele frequency start and end for
##     each SNP
write.table(data.frame(genotype_names, 'effect'=snp_fitness_effects,
                       'initial'=initial_allele_freqs,
                       'final'=allele_freqs,
                       'ld_95'=ld_groups_95,
                       'ld_80'=ld_groups_80),
            file=paste0(output_prefix, '.snp_effects.txt'),
            sep='\t', col.names=TRUE, row.names=FALSE, quote=FALSE)

##  4. Table of strain frequencies and fitnesses
write.table(data.frame(strain=names(fitnesses), 'fitness'=fitnesses,
                       'relative_fitness'=relative_fitnesses,
                       'initial'=initial_strain_freqs,
                       'final'=strain_freqs),
            file=paste0(output_prefix, '.strain_frequencies.txt'),
            sep='\t', col.names=TRUE, row.names=FALSE, quote=FALSE)

## 5. Phenotype file with change in strain frequency
write.table(cbind(names(strain_freqs), names(strain_freqs),
                  strain_freqs - initial_strain_freqs),
            file=paste0(output_prefix, '.strain_freqs.txt'),
            sep='\t', col.names=FALSE, row.names=FALSE, quote=FALSE)


#' Markov Chain Monte Carlo
#'
#' A flexible implementation of the Metropolis-Hastings MCMC algorithm.
#'
#' @param fun A function. Returns the log-likelihood.
#' @param initial Either a numeric matrix or vector, or an object of class [coda::mcmc]
#' or [coda::mcmc.list] (see details).
#' initial values of the parameters for each chain (See details).
#' @param nsteps Integer scalar. Length of each chain.
#' @param nchains Integer scalar. Number of chains to run (in parallel).
#' @param cl A `cluster` object passed to [parallel::clusterApply].
#' @param thin Integer scalar. Passed to [coda::mcmc].
#' @param kernel An object of class [fmcmc_kernel].
#' @param burnin Integer scalar. Length of burn-in. Passed to
#' [coda::mcmc] as \code{start}.
#' @param multicore Logical. If `FALSE` then chains will be executed in serial.
#' @param ... Further arguments passed to \code{fun}.
#' @param conv_checker A function that receives an object of class [coda::mcmc.list],
#' and returns a logical value with `TRUE` indicating convergence.
#'
#' @details This function implements MCMC using the Metropolis-Hastings ratio with
#' flexible transition kernels. Users can specify either one of the available
#' transition kernels or define one of their own (see [kernels]). Furthermore,
#' it allows easy parallel implementation running multiple chains in parallel. In
#' addition, we incorporate a variety of convergence diagnostics, alternatively
#' the user can specify their own (see [convergence-checker]).
#'
#' We now give details of the various options included in the function.
#'
#' @section Starting point:
#'
#' By default, if `initial` is of class `mcmc`, `MCMC` will take the last `nchains`
#' points from the chain as starting point for the new sequence. If `initial` is
#' of class `mcmc.list`, the number of chains in `initial` must match the `nchains`
#' parameter.
#'
#' If `initial` is a vector, then it must be of length equal to the number of
#' parameters used in the model. When using multiple chains, if `initial` is not
#' an object of class `mcmc` or `mcmc.list`, then it must be a numeric matrix
#' with as many rows as chains, and as many columns as parameters in the model.
#'
#' @section Multiple chains:
#'
#' When \code{nchains > 1}, the function will run multiple chains. Furthermore,
#' if \code{cl} is not passed, \code{MCMC} will create a \code{PSOCK} cluster
#' using [parallel::makePSOCKcluster] with
#' [parallel::detectCores]
#' clusters and attempt to execute using multiple cores. Internally, the function does
#' the following:
#'
#' \preformatted{
#'   # Creating the cluster
#'   ncores <- parallel::detectCores()
#'   ncores <- ifelse(nchains < ncores, nchains, ncores)
#'   cl     <- parallel::makePSOCKcluster(ncores)
#'
#'   # Loading the package and setting the seed using clusterRNGStream
#'   invisible(parallel::clusterEvalQ(cl, library(fmcmc)))
#'   parallel::clusterSetRNGStream(cl, .Random.seed)
#' }
#'
#' When running in parallel, objects that are
#' used within \code{fun} must be passed through \code{...}, otherwise the cluster
#' will return with an error.
#'
#' The user controls the initial value of the parameters of the MCMC algorithm
#' using the argument `initial`. When using multiple chains, i.e., `nchains > 1`,
#' the user can specify multiple starting points, which is recommended. In such a
#' case, each row of `initial` is use as a starting point for each of the
#' chains. If `initial` is a vector and `nchains > 1`, the value is recycled, so
#' all chains start from the same point (not recommended, the function throws a
#' warning message).
#'
#' @section Automatic stop:
#'
#' By default, no automatic stop is implemented. If one of the functions in
#' [convergence-checker] is used, then the MCMC is done by bulks as specified
#' by the convergence checker function, and thus the algorithm will stop if,
#' the `conv_checker` returns `TRUE`. For more information see [convergence-checker].
#'
#' @return An object of class [coda::mcmc] from the \CRANpkg{coda}
#' package. The \code{mcmc} object is a matrix with one column per parameter,
#' and \code{nsteps} rows. If \code{nchains > 1}, then it returns a [coda::mcmc.list].
#'
#' @export
#' @examples
#' # Univariate distributed data with multiple parameters ----------------------
#' # Parameters
#' set.seed(1231)
#' n <- 1e3
#' pars <- c(mean = 2.6, sd = 3)
#'
#' # Generating data and writing the log likelihood function
#' D <- rnorm(n, pars[1], pars[2])
#' fun <- function(x) {
#'   x <- log(dnorm(D, x[1], x[2]))
#'   sum(x)
#' }
#'
#' # Calling MCMC, but first, loading the coda R package for
#' # diagnostics
#' library(coda)
#' ans <- MCMC(
#'   fun, initial = c(mu=1, sigma=1), nsteps = 2e3,
#'   kernel = kernel_reflective(scale = .1, ub = 10, lb = 0)
#'   )
#'
#' # Ploting the output
#' oldpar <- par(no.readonly = TRUE)
#' par(mfrow = c(1,2))
#' boxplot(as.matrix(ans),
#'         main = expression("Posterior distribution of"~mu~and~sigma),
#'         names =  expression(mu, sigma), horizontal = TRUE,
#'         col  = blues9[c(4,9)],
#'         sub = bquote(mu == .(pars[1])~", and"~sigma == .(pars[2]))
#' )
#' abline(v = pars, col  = blues9[c(4,9)], lwd = 2, lty = 2)
#'
#' plot(apply(as.matrix(ans), 1, fun), type = "l",
#'      main = "LogLikelihood",
#'      ylab = expression(L("{"~mu,sigma~"}"~"|"~D))
#' )
#' par(oldpar)
#'
#' \dontrun{
#' # In this example we estimate the parameter for a dataset with ----------------
#' # With 5,000 draws from a MVN() with parameters M and S.
#'
#' # Loading the required packages
#' library(mvtnorm)
#' library(coda)
#'
#' # Parameters and data simulation
#' S <- cbind(c(.8, .2), c(.2, 1))
#' M <- c(0, 1)
#'
#' set.seed(123)
#' D <- rmvnorm(5e3, mean = M, sigma = S)
#'
#' # Function to pass to MCMC
#' fun <- function(pars) {
#'   # Putting the parameters in a sensible way
#'   m <- pars[1:2]
#'   s <- cbind( c(pars[3], pars[4]), c(pars[4], pars[5]) )
#'
#'   # Computing the unnormalized log likelihood
#'   sum(log(dmvnorm(D, m, s)))
#' }
#'
#' # Calling MCMC
#' ans <- MCMC(
#'   initial = c(mu0=5, mu1=5, s0=5, s01=0, s2=5),
#'   fun,
#'   kernel  = kernel_reflective(
#'     lb    = c(-10, -10, .01, -5, .01),
#'     ub    = 5
#'     scale = 0.01
#'   ),
#'   nsteps  = 1e5,
#'   thin    = 20,
#'   burnin  = 5e3
#' )
#'
#' # Checking out the outcomes
#' plot(ans)
#' summary(ans)
#'
#' # Multiple chains -----------------------------------------------------------
#'
#' # As we want to run -fun- in multiple cores, we have to
#' # pass -D- explicitly (unless using Fork Clusters)
#' # just like specifying that we are calling a function from the
#' # -mvtnorm- package.
#'
#' fun <- function(pars, D) {
#'   # Putting the parameters in a sensible way
#'   m <- pars[1:2]
#'   s <- cbind( c(pars[3], pars[4]), c(pars[4], pars[5]) )
#'
#'   # Computing the unnormalized log likelihood
#'   sum(log(mvtnorm::dmvnorm(D, m, s)))
#' }
#'
#' # Two chains
#' ans <- MCMC(
#'   initial = c(mu0=5, mu1=5, s0=5, s01=0, s2=5),
#'   fun,
#'   nchains = 2,
#'   kernel  = kernel_reflective(
#'     lb    = c(-10, -10, .01, -5, .01),
#'     ub    = 5
#'     scale = 0.01
#'   ),
#'   nsteps  = 1e5,
#'   thin    = 20,
#'   burnin  = 5e3,
#'   D       = D
#' )
#'
#' summary(ans)
#' }
#'
#' @aliases Metropolis-Hastings
MCMC <- function(
  initial,
  fun,
  nsteps,
  ...,
  nchains      = 1L,
  burnin       = 0L,
  thin         = 1L,
  kernel       = kernel_normal(),
  multicore    = FALSE,
  conv_checker = NULL,
  cl           = NULL
) UseMethod("MCMC")

#' @export
#' @rdname MCMC
MCMC.mcmc <- function(
  initial,
  fun,
  nsteps,
  ...,
  nchains      = 1L,
  burnin       = 0L,
  thin         = 1L,
  kernel       = kernel_normal(),
  multicore    = FALSE,
  conv_checker = NULL,
  cl           = NULL
) {

  MCMC.default(
    initial      = utils::tail(initial, nchains - 1L),
    fun          = fun,
    nsteps       = nsteps,
    ...,
    nchains      = nchains,
    burnin       = burnin,
    thin         = thin,
    kernel       = kernel,
    multicore    = multicore,
    conv_checker = conv_checker,
    cl           = cl
  )

}

#' @export
#' @rdname MCMC
MCMC.mcmc.list <- function(
  initial,
  fun,
  nsteps,
  ...,
  nchains      = 1L,
  burnin       = 0L,
  thin         = 1L,
  kernel       = kernel_normal(),
  multicore    = FALSE,
  conv_checker = NULL,
  cl           = NULL
) {

  if (nchains != length(initial))
    stop(
      "The parameter `nchains` must equal the number of chains passed by ",
      "`initial`.", call. = FALSE
      )

  MCMC.default(
    initial      = do.call(rbind, utils::tail(initial, 0)),
    fun          = fun,
    nsteps       = nsteps,
    ...,
    nchains      = nchains,
    burnin       = burnin,
    thin         = thin,
    kernel       = kernel,
    multicore    = multicore,
    conv_checker = conv_checker,
    cl           = cl
  )

}

#' @export
#' @rdname MCMC
MCMC.default <- function(
  initial,
  fun,
  nsteps,
  ...,
  nchains      = 1L,
  burnin       = 0L,
  thin         = 1L,
  kernel       = kernel_normal(),
  multicore    = FALSE,
  conv_checker = NULL,
  cl           = NULL
  ) {

  # # if the coda package hasn't been loaded, then return a warning
  # if (!("package:coda" %in% search()))
  #   warning("The -coda- package has not been loaded.", call. = FALSE, )

  # Checking initial argument
  initial <- check_initial(initial, nchains)

  if (multicore && nchains == 1L)
    stop("When `multicore = TRUE`, `nchains` should be greater than 1.",
         call. = FALSE)

  if (nchains < 1L)
    stop("`nchains` must be an integer greater than 1.", call. = FALSE)

    # Checkihg burnins
  if (burnin >= nsteps)
    stop("-burnin- (",burnin,") cannot be >= than -nsteps- (",nsteps,").", call. = FALSE)

  # Checking thin
  if (thin >= nsteps)
    stop("-thin- (",thin,") cannot be > than -nsteps- (",nsteps,").", call. = FALSE)

  if (thin < 1L)
    stop("-thin- should be >= 1.", call. = FALSE)

  # Filling the gap on parallel
  if (multicore && !length(cl)) {

    # Creating the cluster
    ncores <- parallel::detectCores()
    ncores <- ifelse(nchains < ncores, nchains, ncores)
    cl     <- parallel::makePSOCKcluster(ncores)

    # Loading the package and setting the seed using clusterRNGStream
    invisible(parallel::clusterEvalQ(cl, library(fmcmc)))
    parallel::clusterSetRNGStream(cl, .Random.seed)

    on.exit(parallel::stopCluster(cl))
  }

  if (nchains > 1L) {

    # Preparing the call for multicore
    fmcmc_call <- as.call(
      c(
        if (multicore)
          list(quote(parallel::clusterApply), cl=quote(cl), x = quote(1L:nchains))
        else
          list(quote(lapply), X = quote(1L:nchains)),
        list(
          FUN = quote(function(
            i, fun., initial., nsteps., thin., kernel., burnin., ...) {

          MCMC(
            fun          = fun.,
            ...,
            initial      = initial.[i, , drop = FALSE],
            nsteps       = nsteps.,
            burnin       = burnin.,
            thin         = thin.,
            kernel       = kernel.,
            nchains      = 1L,
            multicore    = FALSE,
            cl           = NULL,
            conv_checker = NULL
            )

        }),
        fun.     = quote(fun),
        initial. = quote(initial),
        nsteps.  = quote(nsteps),
        burnin.  = quote(burnin),
        thin.    = quote(thin),
        kernel.  = quote(kernel),
        quote(...)
        )
      )
    )

    # updating names
    if (multicore)
      names(fmcmc_call)[names(fmcmc_call) == "FUN"] <- "fun"

    fmcmc_call <- as.call(list(quote(do.call),quote(coda::mcmc.list), fmcmc_call))

  } else if (!is.null(conv_checker)) {

    # If not multicore, still we need to make sure that we are passing some
    # variables as symbols and not as constants. As in the convergence checker
    # function we modify the current environment in order to adapt the algorithm.
    fmcmc_call              <- match.call()
    fmcmc_call$fun          <- quote(fun)
    fmcmc_call$nsteps       <- quote(nsteps)
    fmcmc_call$thin         <- quote(thin)
    fmcmc_call$burnin       <- quote(burnin)
    fmcmc_call$conv_checker <- enquote(NULL)

  }

  # If conv_checker, we run it with the conv checker and return. Notice that
  # we already adapted the code for the case in which we are runing multiple
  # chains
  if (!is.null(conv_checker)) {

    fmcmc_call <- call(
      "with_autostop",
      fmcmc_call,
      conv_checker = quote(conv_checker)
      )

    ans <- eval(fmcmc_call)
    return(ans)

  # If we are not using conv_checker, but still have multiple chains, then
  # we still have to run this somewhat recursively.
  } else if (nchains > 1L) {
    ans <- eval(fmcmc_call)
    return(ans)
  }

  # Adding names
  initial <- initial[1,,drop=TRUE]
  cnames  <- names(initial)

  # Wrapping function. If ellipsis is there, it will wrap it
  # so that the MCMC call only uses a single argument
  passedargs <- names(list(...))
  # print(match.call())
  funargs    <- methods::formalArgs(eval(fun))

  # Compiling
  # cfun <- compiler::cmpfun(fun)

  # ... has extra args
  if (length(passedargs)) {
    # ... has stuff that fun doesnt
    if (any(!(passedargs %in% funargs))) {

      stop("The following arguments passed via -...- are not present in -fun-:\n - ",
           paste(setdiff(passedargs, funargs), collapse=",\n - "),".\nThe function",
           "was expecting:\n - ", paste0(funargs, collapse=",\n - "), ".", call. = FALSE)

    # fun has stuff that ... doesnt
    } else if (length(funargs) > 1 && any(!(funargs[-1] %in% passedargs))) {

      stop("-fun- requires more arguments to be passed via -...-.", call. = FALSE)

    # Everything OK
    } else {

      f <- function(z) {
        fun(z, ...)
      }

    }
  # ... doesnt have extra args, but funargs does!
  } else if (length(funargs) > 1) {

    stop("-fun- has extra arguments not passed by -...-.", call. = FALSE)

  # Everything OK
  } else {

    f <- function(z) fun(z)

  }

  # MCMC algorithm -----------------------------------------------------------

  theta0 <- initial
  theta1 <- theta0
  f0     <- f(theta0)
  f1     <- f(theta1)

  # The updates can be done jointly or sequentially
  klogratio <- kernel$logratio(environment())
  if (length(klogratio) > 1L) {
    joint_rate <- FALSE
    R   <- matrix(stats::runif(nsteps * length(initial)), nrow = nsteps)
    R[] <- log(R)
  } else {
    joint_rate <- TRUE
    R <- matrix(log(stats::runif(nsteps)), nrow = nsteps)
  }

  ans <- matrix(ncol = length(initial), nrow = nsteps,
                dimnames = list(1:nsteps, cnames))

  for (i in 1L:nsteps) {
    # Step 1. Propose
    theta1[] <- kernel$proposal(environment())
    f1       <- f(theta1)

    # Checking f(theta1) (it must be a number, can be Inf)
    if (is.nan(f1) | is.na(f1) | is.null(f1))
      stop(
        "fun(par) is undefined (", f1, ")",
        "Check either -fun- or the -lb- and -ub- parameters.",
        call. = FALSE
      )

    # Step 2. Hastings ratio
    klogratio <- kernel$logratio(environment())
    if (joint_rate) {

      if (R[i] < klogratio) {
        theta0 <- theta1
        f0     <- f1
      }

    } else {

      klogratio <- (R[i, ] < klogratio)
      theta0[klogratio] <- theta1[klogratio]

    }

    # Step 3. Saving the state
    ans[i,] <- theta0

  }

  # Thinning the data
  if (burnin) ans <- ans[-c(1:burnin), , drop = FALSE]
  if (thin)   ans <- ans[(1:nrow(ans) %% thin) == 0, , drop = FALSE]

  # Returning an mcmc object from the coda package
  return(
    coda::mcmc(
      ans,
      start = as.integer(rownames(ans)[1]),
      end   = as.integer(rownames(ans)[nrow(ans)]),
      thin  = thin
      )
    )

}


# Function to fit continous heavy tail functions
# Mostly using poweRlaw package
#
fit_con_heavy_tail <- function (data_set,xmins,options.output)
{
	n <- length(unique(data_set))
	n_models <- 4
	n_sets <- 1 # delete set 3
	# List of models
	model_list = list(list(model=vector("list", length=0),
						   						GOF=vector("list", length=0),
												 xmin_estimation=vector("list", length=0),
												 uncert_estimation=vector("list", length=0),
												 k=0,
												 LL=0,
												 n=n,
												 AICc=0,
												 delta_AICc=0,
												 AICc_weight=0,
						   model_name=character(0),
						   model_set=character(0)))
	fit_ht <- rep(model_list,n_models*n_sets)
	dim(fit_ht) <- c(n_models,n_sets)
	# Two sets: power law with and without cut-off
	# Declare models
	for (i_set in 1:(n_sets)){
		# Continuous power law
		fit_ht[[1,i_set]]$model <- conpl$new(data_set)
		fit_ht[[1,i_set]]$k <- 1
		# Continuous log-normal
		fit_ht[[2,i_set]]$model <- conlnorm$new(data_set)
		fit_ht[[2,i_set]]$k <- 2
		# Continuous exponential
		fit_ht[[3,i_set]]$model <- conexp$new(data_set)
		fit_ht[[3,i_set]]$k <- 1
		# power law with exponential cutoff
		fit_ht[[4,i_set]]$model <- ""
		fit_ht[[4,i_set]]$k <- 2
	}

	# Set xmin for set 1
	#fit_ht[[1,1]]$model$xmin <- 9

	# Estimate Xmin with complete data_set for power law model
	fit_ht[[1,1]]$xmin_estimation <- estimate_xmin(fit_ht[[1,1]]$model,
		xmins = xmins,
		pars = NULL,
		xmax = max(data_set))

	fit_ht[[1,1]]$model$setXmin(fit_ht[[1,1]]$xmin_estimation)

	model_names <- c("Power", "LogNorm","Exp","PowerExp")
	labels_set <- c("Estimated Xmin")

	AICc_weight <- matrix( nrow = n_models, ncol = n_sets,
							dimnames = list(model_names,labels_set))

	delta_AICc <- AICc_weight
	GOF <- delta_AICc
	if(n>8) {
		for (i_set in 1:n_sets){
		aic_min=Inf
		norm_aic_weight=0
		for (i in 1:(n_models)){
			# Set cut-off (x_min)
			fit_ht[[i,i_set]]$model$xmin <- fit_ht[[1,i_set]]$model$xmin

			# Correct n with xmin
			fit_ht[[i,i_set]]$n <-length(data_set[data_set>=fit_ht[[1,i_set]]$model$xmin])

			# Fit models
			#
			if(i!=n_models){
				fit_ht[[i,i_set]]$model$setPars(estimate_pars(fit_ht[[i,i_set]]$model))
				#
				# Get Loglikelihood
				fit_ht[[i,i_set]]$LL <- dist_ll(fit_ht[[i,i_set]]$model)
			} else {
				# Fit Power Exponential (not poweRlaw package)
				#
				fit_ht[[i,i_set]]$model<- powerexp.fit(data_set,fit_ht[[1,i_set]]$model$xmin)
				#
				# Get Loglikelihood
				fit_ht[[i,i_set]]$LL <- fit_ht[[i,i_set]]$model$loglike
			}
			LL <- fit_ht[[i,i_set]]$LL
			k <- fit_ht[[i,i_set]]$k
			# Compute AICc
			#
			fit_ht[[i,i_set]]$AICc <- (2*k-2*LL)+2*k*(k+1)/(n-k-1)
			aic_min <- min(aic_min,fit_ht[[i,i_set]]$AICc)
			if (i==1 & options.output$GOF){
				#
				# Goodness of fit via boostrap
				#
				# xmins is fixed at estimated value
				fit_ht[[i,i_set]]$GOF <- bootstrap_p(fit_ht[[i,i_set]]$model,
											 xmins=fit_ht[[i,i_set]]$model$xmin,
											 pars = NULL,
											 xmax = max(data_set),
											 no_of_sims = 1000,
											 threads = parallel::detectCores())
				#
				# Uncertainty in parms estimation via bootstrap
				#
				# Range of xmins to make bootstrap
				l_xmin <- round(fit_ht[[i,i_set]]$model$xmin * 0.8)
				u_xmin <- round(fit_ht[[i,i_set]]$model$xmin * 1.2)

				fit_ht[[i,i_set]]$uncert_estimation <- bootstrap(fit_ht[[i,i_set]]$model,
											 xmins=l_xmin:u_xmin,
											 pars = NULL,
											 xmax = max(data_set),
											 no_of_sims = 1000,
											 threads = parallel::detectCores())
			}
			fit_ht[[i,i_set]]$model_set <- labels_set[i_set]
			fit_ht[[i,i_set]]$model_name <- model_names[i]
		}

		for (i in 1:n_models){
			delta_AICc[i,i_set] <- fit_ht[[i,i_set]]$AICc - aic_min
			fit_ht[[i,i_set]]$delta_AICc <- delta_AICc[i,i_set]
			norm_aic_weight <- norm_aic_weight + exp(-0.5*fit_ht[[i,i_set]]$delta_AICc)
		}
		# Akaike weigths
		for (i in 1:n_models){
			AICc_weight[i,i_set]  <- exp(-0.5*fit_ht[[i,i_set]]$delta_AICc)/norm_aic_weight
			fit_ht[[i,i_set]]$AICc_weight <- AICc_weight[i,i_set]
		}
		# Plots
		if (options.output$ploting){
			#setwd(options.output$resultsDir)
			fnam <-paste0(strsplit(options.output$data_set_name[i_im],".tif"),"_",labels_set[i_set], ".png")
			png(filename=fnam, res=300,units = "mm", height=200, width=200,bg="white")

			po <-plot(fit_ht[[1,i_set]]$model,xlab="Area",ylab="CCDF",main=labels_set[i_set])
			for (i in 1:n_models){
				if(i!=n_models) {
					lines(fit_ht[[i,i_set]]$model, col=i+1)
				} else {
					est1 <- fit_ht[[i,i_set]]$model
					x <- sort(unique(data_set))
					x <- x[x>=est1$xmin]
					shift <- max(po[po$x>=est1$xmin,]$y)
					y <- ppowerexp(x,est1$xmin,est1$exponent,est1$rate,lower.tail=F)*shift
					lines(x,y,col=i+1)
				}
			}

			legend("topright",model_names,bty="n",col=seq(2,5),lty=c(1,1,1,1),cex=0.5)
			dev.off()
		}
	}
	}
	return(list(fitted_models=fit_ht,delta_AICc=delta_AICc,AICc_weight=AICc_weight))
}

# Function to extract a data frame from output generated by fit_con_heavy_tail
#
#
extract_fit_ht <-function(e)
{
	ee <- e$model
	gg <- e$GOF
	uu <- e$uncert_estimation
	bootXmin <-list(quantile(uu$bootstraps[,2],probs=c(0.025,0.25,0.5,0.75,0.975),na.rm=T))
	bootPar1 <-list(quantile(uu$bootstraps[,3],probs=c(0.025,0.25,0.5,0.75,0.975),na.rm=T))

	if(grepl("con",class(ee),fixed = T))
	{
# 		if(class(ee)=="conlnorm") {
# 			pa<-ee$getPars()
# 			if(length(pa)!=2) stop("conlnorm must have 2 parameters")
# 		} else {
# 			pa<-double(2)
# 			pa[1]<- ee$getPars()
# 			pa[2]<- 0
# 		}
		data_frame(model_name=e$model_name,model_set=e$model_set,
				   par1=ifelse(is.null(ee$pars[1]),NA,ee$pars[1]),
				   par2=ifelse(is.null(ee$pars[2]),NA,ee$pars[2]),
				   xmin=ee$getXmin(),n=e$n,LL=e$LL,AICc=e$AICc,delta_AICc=e$delta_AICc,
				   AICc_weight=e$AICc_weight,GOFp=ifelse(is.null(gg$p),NA,gg$p),bXminQuant=bootXmin,bPar1Quant=bootPar1)

	} else {
		data_frame(model_name=e$model_name,model_set=e$model_set,par1=ee$exponent,par2=ee$rate,
				   xmin=ee$xmin,n=e$n,LL=e$LL,AICc=e$AICc,delta_AICc=e$delta_AICc,
				   AICc_weight=e$AICc_weight,GOFp=ifelse(is.null(gg$p),NA,gg$p),bXminQuant=bootXmin,bPar1Quant=bootPar1)
	}

}

# Read data to call fit_con_heavy_tail and format output in a data.frame
#
#
call_fit_con_heavy_tail <-function(options,i){

	# Read binary data
	connection_file <- file(options$original_bin_files[i], "rb")
	data_set <- readBin(connection_file, "double", n = 10^6)

	ta<-sum(data_set)*233*233/1000000

	cat(options$original_bin_files[i],ta,"\n")

	fit_ht_df<-data_frame()

	if(ta>100000){# Set folder with results for the region

		#
		# Sample data for testing
		if(options$sample_data>0) data_set<- sample(data_set,options$sample_data) ### TESTING

		# Fit distribution models
		fit_ht <- fit_con_heavy_tail(data_set,options$xmins,options)

		# List to data.frame
		fit_ht_df <- ldply(fit_ht$fitted_models,extract_fit_ht)
		fit_ht_df$data_set_name <- strsplit(options$data_set_name[i],".tif")[[1]][1]
		ss <- strsplit(options$data_set_name[i],"_")
		fit_ht_df$region <- ss[[1]][2]
		fit_ht_df$subregion <- ss[[1]][3]
		fit_ht_df$threshold <- ss[[1]][4]
		fit_ht_df$year <- gsub(".*\\.A([0-9]{4}).*","\\1",options$data_set_name[i])
	}

	close(connection_file)

	return(fit_ht_df)

}

# Returns a data.frame with: number of patches, max patch, and total patch area
#
data_con_heavy_tail <-function(options,i){

	# Read binary data
	connection_file <- file(options$original_bin_files[i], "rb")
	data_set <- readBin(connection_file, "double",n = 10^7)
	if(length(data_set)==10^7)
			warning("Data set",options$original_bin_files[i], " may be bigger than 10^7")
	nl<-length(data_set)
	mx<-max(data_set)*233*233/1000000 # Convert to km2
	ta<-sum(data_set)*233*233/1000000
	mx2 <- (sort(data_set,partial=nl-1)[nl-1])*233*233/1000000
	cat(options$original_bin_files[i],"\tNo.patches",nl,"\tMax.PatchKm2",mx,"\t2nd.Max.PatchKm2",mx2,"\tTotalKm2",ta,"\n")

	fit_ht_df <- data_frame(number_of_patches=nl,max_patch=mx,second_max=mx2,total_patch_area=ta)
	fit_ht_df$data_set_name <- strsplit(options$data_set_name[i],".tif")[[1]][1]
	ss <- strsplit(options$data_set_name[i],"_")
	fit_ht_df$region <- ss[[1]][2]
	fit_ht_df$subregion <- ss[[1]][3]
	fit_ht_df$threshold <- ss[[1]][4]
	fit_ht_df$year <- gsub(".*\\.A([0-9]{4}).*","\\1",options$data_set_name[i])

	close(connection_file)

	return(fit_ht_df)

}

# Fit all the images patch sizes for a region with maybe different areas
#
#
region_fit_con_heavy_tail <-function(options,region,year=0){

	# Change to results folder
	#
	setwd(options$resultsDir)

	# Get files with patch sizes (*.bin) and image file names *.tif (data_set_name)
	#
	options$original_bin_files <- list.files(pattern=paste0("^.*",region,".*\\.bin$")) # list.files(pattern="*.\\.bin")
	options$data_set_name <- unlist(strsplit(options$original_bin_files,".bin"))

	if( year > 0)
		options$data_set_name <- options$data_set_name[gsub(".*\\.A([0-9]{4}).*","\\1",options$data_set_name)==year]

	fit <- data_frame()

	for(i in 1:length(options$data_set_name))
	{
		if(options$fit)
		{
		  	fit <- rbind(fit,call_fit_con_heavy_tail(options,i))	# Fit models
		} else {
		 	fit <- rbind(fit,data_con_heavy_tail(options,i)) 		# calculate patch stats
		}
	}
	if(region!=fit$region[1]) warning("Regions don't match ",region, fit$region)

	# Change to base folder
	#
	setwd(oldcd)

	return(fit)
}

# Plot of frequencies of patch sizes with fitted continuous heavy tail functions
# x: data
# fit_ht_df : dataframe with fitted parameters
#
freq_plot_con_ht <- function(x,fit_ht,tit="") # PLOT ALL FROM X=1 ?????????????????
{
	require(dplyr)
	require(ggplot2)
	xx <-as.data.frame(table(x))
	xx$x <- as.numeric(as.character(xx$x))
	ff <- filter(fit_ht,model_name=="PowerExp")
	xmin <- ff$xmin
	xx$pexp <-dpowerexp(xx$x,1,ff$par1,ff$par2)

	ff <- filter(fit_ht,model_name=="Power")
	xx$pow  <-dpareto(xx$x,xmin,ff$par1)

	ff <- filter(fit_ht,model_name=="Exp")
	xx$exp  <-dexp(xx$x,ff$par1)

	xx <-mutate(xx,pexp=ifelse(x<xmin,NA,pexp),pow=ifelse(x<xmin,NA,pexp),
				exp=ifelse(x<xmin,NA,exp),
				Freq=Freq/sum(Freq))

	minFreq <- min(xx$Freq) - 0.5*min(xx$Freq)
	minFreq <- ifelse(minFreq<0,0,minFreq)
	g <- ggplot(xx, aes(y=Freq,x=x)) +  theme_bw() + geom_point(alpha=0.3) +
		coord_cartesian(ylim=c(1,min(minFreq)))+
		scale_y_log10() +scale_x_log10() + ylab("Frequency") + xlab("Patch size") +ggtitle(tit)

	mc <- c("#E69F00", "#56B4E9", "#009E73", "#F0E442", "#0072B2", "#D55E00", "#CC79A7")

	g <- g + geom_line(aes(y=pow,x=x,colour="P.law")) +
		geom_line(aes(y=pexp,x=x,colour="P.law with\nexp. cutoff"))+
		geom_line(aes(y=exp,x=x,colour="Exp."))+
		scale_colour_manual(values=mc,name="")

	fil <- gsub(" ", "", tit, fixed = TRUE)
	fil <- paste0(fil,".png")
	if(tit=="")
		print(g)
	else
		ggsave(fil,plot=g,width=6,height=4,units="in",dpi=600)

}

# Function to plot CCDF of fitted continuous heavy tail functions using ggplo2
#
# x: data
# fit_ht : dataframe with fitted parameters
# tit: file name to save graph
# fit_ht1: second set of parameters to superimpose in the same graph

cdfplot_conpl_exp <- function(x,fit_ht,tit="",xmax=0)
{
	require(poweRlaw)
	m <- conpl$new(x)
	tP <- plot(m,draw=F)
	require(ggplot2)
	require(dplyr)

	ff <- filter(fit_ht,model_name=="Power")
	xmin <- ff$xmin

	tP1 <- cdfplot_conpl_exp_helper(x,tP,fit_ht,xmin)

	#tP2 <-filter(tP2, powl>= min(tP$Rank))
	#tP1 <-filter(tP1, powl>= min(tP$Rank))

	#mc <- c("#E69F00", "#56B4E9", "#009E73","#F0E442", "#0072B2","#D55E00", "#CC79A7")
	# Brewer
	#mc <- c("#d7191c","#fdae61","#abd9e9","#2c7bb6")

	g <- ggplot(tP, aes(x=x,y=y)) +  theme_bw() + geom_point(alpha=0.3) +
		coord_cartesian(ylim=c(1,min(tP$y)))+
		scale_y_log10() +scale_x_log10() + ylab("log[P(X > x)]") + xlab("Patch size") #+ggtitle(tit)
	if(xmax>0) {
		g<-g + xlim(0,xmax+1) + scale_x_log10()
	}
	brk<-unique(tP1$model)
	g <- g + geom_line(data=tP1,aes(y=powl,x=psize,colour=model)) +
		#scale_colour_discrete(name="",breaks=brk)
		#scale_colour_manual(values=mc,name="",breaks=brk)
		scale_colour_brewer(type="div",palette=7,name="",breaks=brk)

# 	if(!is.null(fit_ht1))
# 	{
# 		#tP <-filter(tP,x<xmin)
# 		#g <- ggplot(tP, aes(x=x,y=y)) +  theme_bw() + geom_point(alpha=0.3) + coord_cartesian(ylim=c(1,min(tP$y)))+
# 		#	scale_y_log10() +scale_x_log10() + ylab("log[P(X > x)]") + xlab("Patch size") #+ggtitle(tit)
#
#
# 		#xmin <- fit_ht1$xmin
# 		tP2 <- cdfplot_conpl_exp_helper(x,tP,fit_ht1,xmin,"lt")
# 		brk<-unique(tP2$model)
#
# 		g <- g + geom_line(data=tP2,aes(y=powl,x=psize,colour=model)) +
# 			#scale_colour_discrete(name="",breaks=brk)
# 			#scale_colour_manual(values=mc,name="",breaks=brk)
# 			scale_colour_brewer(type="div",palette=7,name="",breaks=brk)
#
# 		fil <- gsub(" ", "", tit, fixed = TRUE)
# 		fil <- paste0(fil,"_1.png")
# 		if(tit=="")
# 			print(g)
# 		else
# 			ggsave(fil,plot=g,width=6,height=4,units="in",dpi=600)
#
#
# 	}
	fil <- gsub(" ", "", tit, fixed = TRUE)
	fil <- paste0(fil,".png")
	if(tit=="")
		print(g)
	else
		ggsave(fil,plot=g,width=6,height=4,units="in",dpi=600)

}

cdfplot_conpl_exp_helper <- function(x,tP,fit_ht,xmin,mode="gt")
{
	x1 <- unique(x)

	# Select model and generate a data frame
	#
	ff <- filter(fit_ht,model_name=="PowerExp")

	if(mode=="gt") {
		x1 <- x1[x1>=xmin]
		shift <- max(filter(tP,x>=xmin)$y)
	} else {
		x1 <- x1[x1<xmin]
		#xmin<-1
		#shift <-1
		# First select lower subset the change to the Xmin of this subset
		xmin <- ff$xmin
		shift <- max(filter(tP,x>=xmin)$y)
	}

	tP2 <- data.frame(psize=x1, powl=ppowerexp(x1,xmin,ff$par1,ff$par2,lower.tail=F)*shift,model="PowerExp")

	ff <- filter(fit_ht,model_name=="Power")

	tP1  <-data.frame(psize=x1, powl=ppareto(x1,xmin,ff$par1,lower.tail=F)*shift,model="Power")

	ff <- filter(fit_ht,model_name=="Exp")
	m <- conexp$new(x)
	m$setPars(ff$par1)
	m$setXmin(xmin)
	tP3 <- data.frame(psize=x1,powl=dist_cdf(m,x1,lower_tail=F)*shift,model="Exp")

	ff <- filter(fit_ht,model_name=="LogNorm")
	m <- conlnorm$new(x)
	m$setPars(c(ff$par1,ff$par2))
	m$setXmin(xmin)
	tP4 <- data.frame(psize=x1,powl=dist_cdf(m,x1,lower_tail=F)*shift,model="LogNorm")
	tP1 <- bind_rows(tP1,tP2,tP3,tP4)
}

# Function to plot CCDF of heavy tailed distributions using base graphics
# x: data set
# fit_ht: a data frame with the fitted parameters
#
cdfplot_con_ht <- function(x,fit_ht,fnam="")
{
	require(dplyr)
	if(fnam!=""){
			fnam <-paste0(fnam,".png")
			png(filename=fnam, res=300,units = "mm", height=200, width=200,bg="white")
	}

	#fil <- gsub(" ", "", tit, fixed = TRUE)
	#fil <- paste0(fil,".png")

	mc <- c("#E69F00", "#56B4E9", "#009E73", "#F0E442", "#0072B2", "#D55E00", "#CC79A7")

	require(poweRlaw)
	m <- conpl$new(x)
	po <- plot(m,xlab="Area",ylab="CCDF")
	#x <- x[x>=xmin]

	#x <- unique(x)
	# Plot with base plot and poweRlaw package
	#
	ff <- filter(fit_ht,model_name=="Power")
	xmin <- ff$xmin
	m$setPars(ff$par1)
	m$setXmin(xmin)
	lines(m, col=mc[1])

	shift <- max(filter(po,x>=xmin)$y)
	ff <- filter(fit_ht,model_name=="PowerExp")
	x1 <- sort(unique(x))
	#x1 <- sort(x)
	x1 <- x1[x1>=xmin]
	y1 <- ppowerexp(x1,xmin,ff$par1,ff$par2,lower.tail=F)*shift
	lines(x1,y1,col=mc[2])

	ff <- filter(fit_ht,model_name=="Exp")
	m <- conexp$new(x)
	m$setPars(ff$par1)
	m$setXmin(xmin)
	lines(m, col=mc[3])

	ff <- filter(fit_ht,model_name=="LogNorm")
	m <- conlnorm$new(x)
	m$setPars(c(ff$par1,ff$par2))
	m$setXmin(xmin)
	lines(m, col=mc[4])

	model_names <- c("Power","PowerExp","Exp","LogNorm")
	legend("topright",model_names,bty="n",col=mc[1:4],lty=c(1,1,1,1),cex=0.5)

	if(fnam!="")
		dev.off()

}

# Function to plot distributions fits from the complete dataset Xmin=9
# and the dataset from the estimated Xmin
# calls the function cdfplot_con_ht and use global variables
#
# reg: geographic region {SA,CA,NA,EU, etc}
#
ccdfPlots_one_region <- function(options,region)
{
	for( i_im in 1:length(options$data_set_name))
	{
		connection_file <- file(options$original_bin_files[i_im], "rb")
		data_set <- readBin(connection_file, "double", n = 10^6)
		if(options$sample_data>0) data_set<- sample(data_set,options$sample_data)
		ta<-sum(data_set)*233*233/1000000
		if(ta>100000){
			ss <- strsplit(options$data_set_name[i_im],"_")[[1]][2]
			if( region != ss) warning("Region extracted from file names: ",ss, " Different from parameter: ",region)

			nn <- strsplit(options$original_bin_files[i_im],".tif")[[1]][1]

			# Complete data set (Xmin=1)
			#
			ff <- filter(all_fit,grepl(nn,data_set_name,ignore.case = T),model_set=="Xmin=9",region==region)
			na<-paste0(ff$region[1],"_", ff$subregion[1],"_",ff$year[1],"_",ff$model_set[1])
			cdfplot_conpl_exp(data_set,ff,na)

			# data set< Estimated Xmin (<Xmin)
			#
			ff <- filter(all_fit,grepl(nn,data_set_name,ignore.case = T),model_set=="Estimated Xmin",region==region)
			na<-paste0(ff$region[1],"_", ff$subregion[1],"_",ff$year[1],"_",ff$model_set[1])
			cdfplot_conpl_exp(data_set,ff,na)
		}
	}

}

# Downloada all modis granules for a year in dataDir folder
#
# Returns a data frame with file names and h v year
# use a python module called pyModis
#
down_modis_year <- function(dataDir,year) {

  setwd(dataDir)
  hdfDir <-"hdf"
  if(!dir.exists(hdfDir)) {
    dir.create(hdfDir)
  }
  starty <- sprintf("01-01-%04d", year)
  endy   <- sprintf("31-12-%04d", year)
  system(paste0("modis_download.py -U lsaravia -P Octa2004 -p MOD44B.051 -f ",starty," -e ",endy," hdf"))
  # modis_download.py -U lsaravia -P Octa2004 -p MOD44B.051 -t h18v03 -f 01-01-2015 -e 31-12-2015 -x hdf
  setwd(oldcd)

}

# Downloada modis granules using the filenames contained in Download.txt in dataDir folder
# check that granules are not greater than hmax and less than hmin
# Returns a data frame with file names and h v year
# use a python module called pyModis
#
down_modis <- function(dataDir,hmin,hmax) {

	setwd(dataDir)
  hdfDir <-"hdf"
  if(!dir.exists(hdfDir)) {
    dir.create(hdfDir)
  }

	dn <- read.table("Download.txt",stringsAsFactors = F)

	require(stringr)

	#regexpr("h[0-9]",dn[1,1])

	dn$h=as.numeric(str_sub(dn[,1],18,19))
	dn$v=as.numeric(str_sub(dn[,1],21,22))
	dn$year=str_sub(dn[,1],9,12)

	require(dplyr)

	#
	# For southamerica h in 07-15
	#
	if( !(hmin==0 & hmax==0)) {
	  dn1 <- filter(dn,h>=hmin & h<=hmax)
	} else {
	  dn1 <- dn
	}
	str_sub(dn1$V1,28,40) <- "*"

	write.table(str_c(dn1$V1,"*"),file="MOD44BFiles",row.names = F,col.names = F,quote = F)

	system("modis_download_from_list.py -U lsaravia -P Octa2004 -p MOD44B.051 -f MOD44BFiles hdf ")

	setwd(oldcd)

	return(dn1)
	}

# Make mosaic of hdf files downloaded with down_modis
# in a folder "hdf" with base dataDir, it uses a file called
# "Download.txt" to read the files to compose and the Python PyModis package
#
# dataDir: base folder to output mosaic geotif files
#
#
mosaic_modis <- function(dataDir,hmin,hmax) {

	setwd(dataDir)
	dn <- read.table("Download.txt",stringsAsFactors = F)
	require(stringr)
	dn$h=as.numeric(str_sub(dn[,1],18,19))
	dn$v=as.numeric(str_sub(dn[,1],21,22))
	dn$year=str_sub(dn[,1],9,12)

	require(dplyr)
	dn1 <- filter(dn,h>=hmin & h<=hmax)
	setwd("hdf")
	group_by(dn1,year) %>% do( s=mosaic_by_year(.))

	setwd(oldcd)
}

# helper function for mosaic_modis
#
mosaic_by_year <-function(x) {

  # write list of file names to use in the mosaic
  #
	write.table(x$fname,file="listfileMOD44B.051.txt",quote=F,col.names = F,row.names = F)

  # Output file name
  #
  gname <- paste0('MOD44B.MRTWEB.A',x$year[1],'065.051.Percent_Tree_Cover.tif')

	s <- paste0('modis_mosaic.py -s "1 0 0 0 0 0 0" listfileMOD44B.051.txt -o ', gname)
	system(s)

	return(gname)
}

# Read the h and v limits for modis hdf files to make composite images by continent
#
#
get_modis_limits <-function(region,hmin,hmax){

  setwd("MODIS_tools")
  fn <- paste0("Download_",region,".txt")
  dn <- read.table(fn,stringsAsFactors = F)
  setwd(oldcd)

  require(stringr)

  # regexpr("h[0-9]",dn[1,1])

  dn$h=as.numeric(str_sub(dn[,1],18,19))
  dn$v=as.numeric(str_sub(dn[,1],21,22))
  dn$year=str_sub(dn[,1],9,12)

  # Generate a DF with only the h v needed for a region
  #
  require(dplyr)
  dn <- filter(dn,h>=hmin & h<=hmax)
  m <- matrix(0,nrow=max(dn$v),ncol=max(dn$h))
  m[dn$v,dn$h]<-1
  print(m)
  mod_limits <- dn %>% filter(year==2000) %>% arrange(v,h) %>% select(h,v) %>% mutate( Region=region)

}

#' Make geotiff mosaic of hdf files for a region defined as h & v modis granules
#' in a folder "hdf" within base dataDir
#'
#' @param dataDir Folder where the geotiff file will be generated and where there is
#'                a subfolder hdf with granules
#'
#' @param destDir Destination folder for the generated geotiff files
#'
#' @param granules Data frame with h & v columns identifying granules to be mosaicked, with
#'                 fields h,v,region
#' @param region  region defined in the data frame to generate the geotiff
#'
#' @param year    year of the modis granules to generate the geotiff
#'
#' @return
#' @export
#'
#' @examples
#'
mosaic_modis_region <- function(dataDir,destDir,granules,region,year) {
  oldcd <-getwd()
  setwd(dataDir)

  # read the files in dir 'hdf'
  #
  dn <- list.files("hdf",pattern = "^.*\\.hdf$")
  if(length(dn)==0) stop("The folder hdf is empty")
  require(dplyr)
  dn <- tibble(dn)
  names(dn) <- "fname"

  require(stringr)
  dn$h <- as.numeric(str_match(dn$fname, "h(\\d{2})")[,2])
  dn$v <- as.numeric(str_match(dn$fname, "v(\\d{2})")[,2])
  dn$year <- str_match(dn$fname, "A(\\d{4})")[,2]

  dn1 <- granules %>% filter(Region==region) %>% inner_join(dn)
  setwd("hdf")

  dn2 <- group_by(dn1,year) %>% do( gname=mosaic_by_year(.))
  dn2$gname <-unlist(dn2$gname)

  dn2$toname <- paste0(destDir,"/",dn2$gname)

  # dn2 <- data.frame(gname="MOD44B.MRTWEB.A2015065.051.Percent_Tree_Cover.tif",stringsAsFactors = F)
  file.rename(dn2$gname,dn2$toname)

  setwd(oldcd)
}

# Read data to call fit_con_heavy_tail and format output in a data.frame
#
#
max_patch_size <-function(im_names,opts.out){

	max_s_df <- data_frame()

	for(i in 1:length(im_names))
	{
		# Read binary data
		connection_file <- file(original_bin_files[i], "rb")
		data_set <- readBin(connection_file, "double", n = 10^6)
		if(options.output$sample_data>0) data_set<- sample(data_set,options.output$sample_data) ### TESTING
		max_s_df <- rbind(max_s_df,data_frame( maxS=max(data_set), data_set_name=strsplit(options.output$data_set_name[i],".tif")[[1]][1],
								year=substr(data_set_name,16,19),region=opts.out$region))

	}

	return(max_s_df)
}

read_random_percolation <-function(region) {

	percolation_R_files <- list.files(pattern="^.*_randomized_p_sweep\\.R$")
	df <- data.frame()

	for(i in 1:length(percolation_R_files)){
		ss <- strsplit(percolation_R_files[i],"_")
		#df$region <- ss[[1]][2]
		#df$subregion <- ss[[1]][3]
		if(ss[[1]][2]!=region) warning("Regions don't match ",region, ss[[1]][2])

		source(percolation_R_files[i])
		ll <- length(p_set)
		df1 <- data.frame(region=rep(ss[[1]][2],ll),subregion=rep(ss[[1]][3],ll),correlation_length_km=t(correlation_length_km_set),p_set=t(p_set))
		df1$equivalent_diameter_km <- equivalent_diameter_km
		df1$maximum_length_km <- maximum_length_km
		df1$total_area_km2 <- total_area_km2
		df <- rbind(df,df1)

	}
	return(df)
}

read_region_correlation<-function(region) {

	r_files <- list.files(pattern="^.*_Tree_Cover.tif_Correlation_analisys\\.R$")
	df <- data.frame()

	for(i in 1:length(r_files)){

		ss <- strsplit(r_files[i],"_")
		#df$region <- ss[[1]][2]
		#df$subregion <- ss[[1]][3]
		if(ss[[1]][2]!=region) warning("Regions don't match ",region, ss[[1]][2])

		source(r_files[i])
		df1 <- data.frame(region=ss[[1]][2],subregion=ss[[1]][3],year=gsub(".*\\.A([0-9]{4}).*","\\1",r_files[i]),
			correlation_length_km,maximum_length_km,total_area_km2,equivalent_diameter_km)

		df <- rbind(df,df1)
	}
	return(df)
}

# Estimation of the correlation exponent epsilon ~ (p-pc)^-nu
# based in the potential relationship near the critical point
#
# range: range of p to estimate exponent
# cp: critical point
# df: data frame with data
#
est_correlation_exponent <- function(range,cp,df)
{
	require(ggplot2)

	f_df <- df %>% group_by(region,subregion) %>% filter(p_set>=range[1] & p_set<=range[2]) %>% mutate(p_dif=abs(p_set-cp))

	g <- ggplot(f_df,aes(p_dif,correlation_length_km,colour=subregion))+theme_bw() + geom_point() + scale_y_log10() +scale_x_log10()
	print(g + geom_smooth(method=lm, se=FALSE))    # Don't add shaded confidence region

	mods <- do(f_df,mod=lm(log(correlation_length_km) ~ log(p_dif),data=.),max_corr_length=max(.$correlation_length_km),min_corr_length=min(.$correlation_length_km))
	slop <- mods %>% mutate(log_inter=coef(summary(mod))[1],correl_exp=coef(summary(mod))[2],
						R2 = summary(mod)$r.squared,pvalue_exp=coef(summary(mod))[2,4], #full_coef=coef(summary(.$mod)),
						max_corr_length=max_corr_length,min_corr_length=min_corr_length) %>% select(-mod)

	#slop <- mods %>% do(data.frame(region=.$region, subregion=.$subregion, var = names(coef(.$mod)), coef(summary(.$mod)))) %>% filter(var=="log(p_dif)")
	#R2s <- mods %>% summarise(region=region, subregion=subregion,R2 = summary(mod)$r.squared)
	#slop$R2 <- R2s$R2

	return(slop) # %>% select(-var,-t.value) %>% rename(correl_exp=Estimate,SE=Std..Error,pvalue=Pr...t..))
}

# Calculates critical probability from correlation length based on the estimated exponents using
# function est_correlation_exponent
#
# range: range of p to estimate exponent (not used)
# cp: critical point
# corexp: data frame ouput of the function est_correlation_exponent
# regcor: data frame with data
#
calc_critical_probability <- function(range,cp,corexp,regcor)
{
	allcor <- regcor %>% inner_join(corexp) %>% mutate(p=cp-(correlation_length_km/exp(log_inter))^(1/correl_exp),
													   p=ifelse(p<0,0,ifelse(p>1,1,p)))
#	incor <- allcor %>% filter(correlation_length_km>=min_corr_length,correlation_length_km<=max_corr_length) %>%
#		mutate(pdist=exp((log_inter-log(correlation_length_km))/correl_exp))
#		mutate(p=cp-(correlation_length_km/exp(log_inter))^(1/correl_exp))
#	ltcor <- allcor %>% filter(correlation_length_km<min_corr_length) %>% mutate(p=0.1)
#	gtcor <- allcor %>% filter(correlation_length_km>max_corr_length) %>% mutate(p=0.9)

	return(select(allcor, -(max_corr_length:pvalue_exp)))
}

# Estimation of the critical point using logistic regression
#
#
est_critical_point <- function(rndper)
{
  # Estimate the probability of percolation as correlation_length/equivalent_diameter
  #
	rp <-rndper %>% group_by(region,subregion) 	%>% mutate(p_correlation_length=correlation_length_km/equivalent_diameter_km) %>% mutate(p_correlation_length=ifelse(p_correlation_length>1,1,p_correlation_length))

  # Fit a binomial model to estimate the critical point
	#
	rfit <- rp %>% do(lfit= glm(.$p_correlation_length ~ .$p_set, family=binomial(logit), data = .))
	# slop <- rfit %>% mutate(beta0 = coef(lfit)[1],
	# 						beta1 = coef(lfit)[2],
	# 						pcrit = - beta0 / beta1)

	# Estimate the critical point using a regression of the values around the probability of percolation =0.50
	# p_correlation_length = 0.5
	#
	slop <- rp %>% do(pcrit = est_critical_point_helper(.)) %>% mutate(pcrit=unlist(pcrit))

	# Calculate the predicted values of the binomial regression to plot
	#
	rfit <-rfit %>% inner_join(rp %>% slice(1) %>% transmute(max_cl=equivalent_diameter_km))
	rp1 <-rfit %>% do(data.frame(fit = .$lfit$fitted*.$max_cl))
	rp <- bind_cols(rp, rp1)

	gt <- slop %>% mutate(label = sprintf("p == %.3f", pcrit),x=0.9,y=0.1)
	g <-ggplot(rp,aes(p_set,correlation_length_km))+theme_bw() + geom_point(shape=21) +
		geom_line(aes(x = p_set, y = fit), linetype = 2) +
		geom_vline(aes(xintercept = pcrit),data = slop, color = "blue") +
		geom_text(aes(x,y,label = label) ,data=gt,parse = TRUE ) +
		facet_wrap(region ~ subregion,scales="free_y")
	print(g)

	return(slop)
}

est_critical_point_helper<- function(x)
{
  # lf <-glm(p_correlation_length ~ p_set, family=binomial(logit), data = x)
  # pcrit=- (lf$coefficients[1] / lf$coefficients[2])
  # x$p <-predict(lf,x,type="resp")
  x1 <- x %>% filter(p_correlation_length>0.50) %>% arrange(p_correlation_length) %>% slice(1:10)
  x1 <- bind_rows(x1, x %>% filter(p_correlation_length<=0.50) %>% arrange(desc(p_correlation_length)) %>% slice(1:10))
  pcrit <- approx(x1$p_correlation_length,x1$p_set,xout=0.5)[["y"]]

}

#' Plot estimated correlations lenght with ramdomized correlation lengths
#'
#' @param cpoint Data frame with critical points
#' @param rndcor Data frame with randomized correlation lengths
#' @param corlen Data frame with correlation lengths
#'
plot_corr_length <- function(cpoint,rndcor,corlen)
{
  gt <- cpoint %>% mutate(label = sprintf("p == %.3f", pcrit),x=0.9,y=0.1)
  g <-ggplot(rndcor,aes(p_set,correlation_length_km))+theme_bw() + geom_point(shape=21) +
    geom_vline(aes(xintercept = pcrit),data = cpoint, color = "blue") +
    geom_text(aes(x,y,label = label) ,data=gt,parse = TRUE ) +
    geom_point(aes(),data=corlen)
    facet_wrap(region ~ subregion,scales="free_y")
  print(g)
}

# Calculates the percolation probability using logistic equation parms from est_critical_point
#
# cripoi: critical point and logistic parameters
# regcor: data frame with data
#
calc_critical_probability_logis <- function(cripoi,regcor)
{
	allcor <- regcor %>% inner_join(cripoi) %>% mutate( pl = (-log((max_cl-correlation_length_km)/correlation_length_km) -beta0) / beta1)

	return(select(allcor,-(lfit:max_cl)))

}

# Plot inverse cumulative size distributions fits by region
#
# region
# odir: folder with the size distribution files
#
plot_size_dist_by_region <-function(region,odir,Thres=0,Year=0)
{
	require(ggplot2)

	options.glo$resultsDir <- odir
	options.glo$sample_data <- 0                 # Set >0 For testing

	setwd(options.glo$resultsDir)
	options.glo$original_bin_files <- list.files(pattern=paste0("^.*",region,".*\\.bin$")) # list.files(pattern="*.\\.bin")

	options.glo$data_set_name <- unlist(strsplit(options.glo$original_bin_files,".bin"))
	year <- gsub(".*\\.A([0-9]{4}).*","\\1",options.glo$data_set_name)
	threshold <- sub("^(?:[^_]*_){3}([^_]+).*","\\1",options.glo$data_set_name)

	if(Thres>0)
		options.glo$data_set_name <- options.glo$data_set_name[threshold==thres]

	if(Year>0)
		options.glo$data_set_name <- options.glo$data_set_name[year==Year]

	# Read binary data
	set.seed(seed=0)

	ccdfPlots_one_region(options.glo,region)

	setwd(oldcd)
}

# Fit continuos heavy tail models (used for small data sets not patch size distributions)
#
#
fit_cont_heavy_tail_mdls <- function (data_set,xmins,ploting=TRUE,gof=FALSE)
{
	n <- length(unique(data_set))
	n_models <- 3
	# List of models
	model_list = list(list(model=vector("list", length=0),
						   						GOF=vector("list", length=0),
												 xmin_estimation=vector("list", length=0),
												 uncert_estimation=vector("list", length=0),
												 k=0,
												 LL=0,
												 n=n,
												 AICc=0,
												 delta_AICc=0,
												 AICc_weight=0,
						   model_name=character(0),
						   modelfit_cont_heavy_tail_mdls_set=character(0)))
	fit_ht <- rep(model_list,n_models)
	dim(fit_ht) <- c(n_models)

	# Declare models
	# Continuous power law
	fit_ht[[1]]$model <- conpl$new(data_set)
	fit_ht[[1]]$k <- 1
	# Continuous log-normal
	fit_ht[[2]]$model <- conlnorm$new(data_set)
	fit_ht[[2]]$k <- 2
	# Continuous exponential
	fit_ht[[3]]$model <- conexp$new(data_set)
	fit_ht[[3]]$k <- 1

	# Set xmin for set 1
	fit_ht[[1]]$model$xmin <- xmins
	if(length(xmins)>1){
		# Estimate Xmin with complete data_set for power law model
		fit_ht[[1]]$xmin_estimation <- estimate_xmin(fit_ht[[1]]$model,
			xmins = xmins,
			pars = NULL,
			xmax = max(data_set))
		fit_ht[[1]]$model$setXmin(fit_ht[[1]]$xmin_estimation)
	} else if(xmins-1){
		fit_ht[[1]]$xmin_estimation <- estimate_xmin(fit_ht[[1]]$model,
													 pars = NULL,
													 xmax = max(data_set))
		fit_ht[[1]]$model$setXmin(fit_ht[[1]]$xmin_estimation)
	} else {
		fit_ht[[1]]$model$setXmin(xmins)
	}

	model_names <- c("Power", "LogNorm","Exp")

	AICc_weight <- matrix( nrow = n_models, ncol = 1,
							dimnames = list(model_names))

	delta_AICc <- AICc_weight
	GOF <- delta_AICc
	if(n>8) {
		aic_min=Inf
		norm_aic_weight=0
		for (i in 1:(n_models)){
			# Set cut-off (x_min)
			fit_ht[[i]]$model$xmin <- fit_ht[[1]]$model$xmin

			# Correct n with xmin
			fit_ht[[i]]$n <-length(data_set[data_set>=fit_ht[[1]]$model$xmin])

			# Fit models
			#
			fit_ht[[i]]$model$setPars(estimate_pars(fit_ht[[i]]$model))
			#
			# Get Loglikelihood
			fit_ht[[i]]$LL <- dist_ll(fit_ht[[i]]$model)

			LL <- fit_ht[[i]]$LL
			k <- fit_ht[[i]]$k
			# Compute AICc
			#
			fit_ht[[i]]$AICc <- (2*k-2*LL)+2*k*(k+1)/(n-k-1)
			aic_min <- min(aic_min,fit_ht[[i]]$AICc)
			if (gof && fit_ht[[i]]$n>4){
				#
				# Goodness of fit via boostrap
				#
				# xmins is fixed at estimated value (or 9)
				fit_ht[[i]]$GOF <- try(bootstrap_p(fit_ht[[i]]$model,
											 #xmins=fit_ht[[i]]$model$xmin,
											 pars = NULL,
											 xmax = max(data_set),
											 no_of_sims = 999,
											 threads = parallel::detectCores()))
				GOF <- fit_ht[[i]]$GOF
				if(class(GOF)!="bs_p_xmin"){
					fit_ht[[i]]$GOF <-NULL
				} else {
					fit_ht[[i]]$GOF$p <- sum(GOF$gof >= GOF$bootstraps$gof,na.rm = TRUE)/999
				}
				#
				# Uncertainty in parms estimation via bootstrap
				#
				# Range of xmins to make bootstrap
# 				l_xmin <- fit_ht[[i]]$model$xmin * 0.8
# 				u_xmin <- fit_ht[[i]]$model$xmin * 1.2
# 				fit_ht[[i]]$uncert_estimation <- bootstrap(fit_ht[[i]]$model,
# 											 xmins=l_xmin:u_xmin,
# 											 pars = NULL,
# 											 xmax = max(data_set),
# 											 no_of_sims = 1000,
# 											 threads = parallel::detectCores())
			}
			fit_ht[[i]]$model_name <- model_names[i]
		}

		for (i in 1:n_models){
			delta_AICc[i] <- fit_ht[[i]]$AICc - aic_min
			fit_ht[[i]]$delta_AICc <- delta_AICc[i]
			norm_aic_weight <- norm_aic_weight + exp(-0.5*fit_ht[[i]]$delta_AICc)
		}
		# Akaike weigths
		for (i in 1:n_models){
			AICc_weight[i]  <- exp(-0.5*fit_ht[[i]]$delta_AICc)/norm_aic_weight
			fit_ht[[i]]$AICc_weight <- AICc_weight[i]
		}
		# Plots
		if (ploting){
			#setwd(options.output$resultsDir)
			#fnam <-paste0(strsplit(options.output$data_set_name[i_im],".tif"),"_",labels_set[i_set], ".png")
			#png(filename=fnam, res=300,units = "mm", height=200, width=200,bg="white")

			po <-plot(fit_ht[[1]]$model,xlab="Area",ylab="CCDF")
			for (i in 1:n_models){
				if(model_names[i]!="PowerExp")
				{
					lines(fit_ht[[i]]$model, col=i+1)
				} else {
					est1 <- fit_ht[[i]]$model
					x <- sort(unique(data_set))
					x <- x[x>=est1$xmin]
					shift <- max(po[po$x>=est1$xmin,]$y)
					y <- ppowerexp(x,est1$xmin,est1$exponent,est1$rate,lower.tail=F)*shift
					lines(x,y,col=i+1)
				}
			}

			legend("topright",model_names,bty="n",col=seq(2,5),lty=c(1,1,1,1),cex=0.5)
			#dev.off()
		}
	}
	fit_ht_df <- ldply(fit_ht,extract_fit_ht_mdls)

	return(fit_ht_df)
}

# Function to extract a data frame from output generated by fit_con_heavy_tail
#
#
extract_fit_ht_mdls <-function(e)
{
	ee <- e$model
	gg <- e$GOF
	uu <- e$uncert_estimation
	bootXmin <-list(quantile(uu$bootstraps[,2],probs=c(0.025,0.25,0.5,0.75,0.975),na.rm=T))
	bootPar1 <-list(quantile(uu$bootstraps[,3],probs=c(0.025,0.25,0.5,0.75,0.975),na.rm=T))

	if(grepl("con",class(ee),fixed = T))
	{
		data_frame(model_name=e$model_name,
				   par1=ifelse(is.null(ee$pars[1]),NA,ee$pars[1]),
				   par2=ifelse(is.null(ee$pars[2]),NA,ee$pars[2]),
				   xmin=ee$getXmin(),n=e$n,LL=e$LL,AICc=e$AICc,delta_AICc=e$delta_AICc,
				   AICc_weight=e$AICc_weight,
				   GOFp=ifelse(is.null(gg$p),NA,
				   		ifelse(is.na(gg$p),sum(gg$bootstraps$KS>gg$gof,na.rm=TRUE)/nrow(gg$bootstraps),gg$p)),

				   bXminQuant=bootXmin,bPar1Quant=bootPar1)

	} else {
		data_frame(model_name=e$model_name,model_set=e$model_set,par1=ee$exponent,par2=ee$rate,
				   xmin=ee$xmin,n=e$n,LL=e$LL,AICc=e$AICc,delta_AICc=e$delta_AICc,
				   AICc_weight=e$AICc_weight,GOFp=ifelse(is.null(gg$p),NA,gg$p),bXminQuant=bootXmin,bPar1Quant=bootPar1)
	}

}

# Fit all the images patch sizes for a region with maybe different areas
#
#
merge2_region_fit_con_heavy_tail <-function(options,region,new_region){

	# Change to results folder
	#
	setwd(options$resultsDir)

	# If new_region files already exist don't merge
	#
	if(list.files(pattern=paste0("^.*",new_region,".*\\.bin$"))=="") {

		# Get the original files nameswith patch sizes (*.bin) and image file names *.tif (data_set_name)
		#
		options$original_bin_files <- list.files(pattern=paste0("^.*",region,".*\\.bin$")) # list.files(pattern="*.\\.bin")

		options$data_set_name <- unlist(strsplit(options$original_bin_files,".bin"))

		# Merge the two largest patches
		#
		for(i in 1:length(options$data_set_name))
		{
			connection_file <- file(options$original_bin_files[i], "rb")
			data_set <- sort(readBin(connection_file, "double", n = 10^6),decreasing=TRUE)
			data_set[2] <- data_set[1]+data_set[2]
			data_set <- data_set[2:length(data_set)]
			close(connection_file)

			# Change the name of the file to new_region
			#
			options$original_bin_files[i] <- sub(region,new_region,options$original_bin_files[i])
			connection_file <- file(options$original_bin_files[i], "wb")
			writeBin(data_set,connection_file)
			close(connection_file)
		}
	}

	# Get the new files names with merged patches
	#

	options$original_bin_files <- list.files(pattern=paste0("^.*",new_region,".*\\.bin$")) # list.files(pattern="*.\\.bin")

	options$data_set_name <- unlist(strsplit(options$original_bin_files,".bin"))

	fit <- data_frame()

	for(i in 1:length(options$data_set_name))
	{
		if(options$fit)
		{
		  	fit <- rbind(fit,call_fit_con_heavy_tail(options,i))	# Fit models
		} else {
		 	fit <- rbind(fit,data_con_heavy_tail(options,i)) 		# calculate patch stats
		}
	}

	# Change to base folder
	#
	setwd(oldcd)

	return(fit)
}

#  Plot of RSmax by year and threshold
#
#  pst: pstat_threshold data frame
#  regions: vector of regions to include
plot_RSmax_yearThreshold <- function(pst,regions){

	ff1 <- filter(pst,regsub %in% regions)
	ff2<- group_by(ff1,regsub,threshold) %>%  summarise(mprop=mean(prop_max_patch))

	if(length(unique(ff1$regsub))==1)
	{
		require(viridis)
		g <- ggplot(ff1, aes(y=prop_max_patch,x=year,colour=regsub)) +  theme_bw() +
		geom_point(shape=19) + geom_line() + facet_wrap(~threshold) + scale_colour_viridis(discrete = T,guide=F) +
			ggtitle(unique(ff1$regsub)) +  theme(plot.title = element_text(hjust = 0.5)) +
			geom_hline(aes(yintercept=mprop,colour=regsub),ff2,linetype = 2)

	} else {
		pd <-position_dodge(.3)
		g <- ggplot(ff1, aes(y=prop_max_patch,x=year,colour=regsub)) +  theme_bw() +
		geom_point(shape=19,position=pd) + facet_wrap(~threshold) + scale_colour_brewer(palette="Set1",name="Region")  +
		geom_hline(aes(yintercept=mprop,colour=regsub),ff2,linetype = 2)
	}

	print(g + ylab(expression(RS[max]))  +
		theme(axis.text.x = element_text(angle=90,hjust=0))
		)

}

#  Plot fluctuations of RSmax by year and faceted by threshold
#
#  pst: pstat_threshold data frame
#  regions: vector of regions to include
plot_RSmax_Fluctuations_yearThreshold <- function(pst,regions,isSignif,numcol=3){
	require(viridis)
	ff1 <- filter(pst,regsub %in% regions)
	ff1 <- filter(ff1,threshold %in% isSignif) %>% mutate(is_signif=TRUE) %>% bind_rows( filter(ff1,!(threshold %in% isSignif)) %>% mutate(is_signif=FALSE) )
	#
	# Fluctuations around the mean Smax - <Smax>/TotArea
	#
	g <- ggplot(ff1, aes(y=delta_prop_max_patch,x=year,colour=is_signif)) +  theme_bw() +
		geom_point(shape=19,size=1) + facet_wrap(~threshold,ncol=numcol) +
		ylab(expression(Delta~RS[max])) + scale_colour_viridis(discrete = T,guide=F,direction = -1) +
		ggtitle(unique(ff1$regsub)) +
		theme(axis.text.x = element_text(angle=90,hjust=0),plot.title = element_text(hjust = 0.5,size=12)) +
		stat_quantile(quantiles=c(.10,.90),linetype=2)

	print(g)
	return(g)
}

plot_RSmax_Fluctuations_yearThreshold_regions <- function(pst,regions,isSignif,numcol=3){
	require(viridis)
	ff1 <- filter(pst,regsub %in% regions)
	ff2 <- inner_join(ff1,isSignif) %>% mutate(is_signif=TRUE)
	ff3 <- anti_join(ff1 %>% ungroup(),isSignif) %>% mutate(is_signif=FALSE)
	ff1 <- bind_rows(ff2,ff3)
	#%>% bind_rows( filter(ff1,!(threshold %in% isSignif)) %>% mutate(is_signif=FALSE) )
	#
	# Fluctuations around the mean Smax - <Smax>/TotArea
	#
	g <- ggplot(ff1, aes(y=delta_prop_max_patch,x=year,colour=is_signif)) +  theme_bw() +
		geom_point(shape=19) + facet_grid(threshold~regsub) +
		ylab(expression(Delta~RS[max])) + scale_colour_viridis(discrete = T,guide=F,direction = -1) +
		theme(axis.text.x = element_text(angle=90,hjust=0),plot.title = element_text(hjust = 0.5)) +
		stat_quantile(quantiles=c(.10,.90),linetype=2)

	print(g)
}

#  Plot fluctuations of Absolute max patch Smax by year and faceted by threshold
#
#  pst: pstat_threshold data frame
#  regions: vector of regions to include
plot_Smax_Fluctuations_yearThreshold <- function(pst,regions){

	ff1 <- filter(pst,regsub %in% regions)
	#
	# Fluctuations around the mean Smax - <Smax>/TotArea
	#
	g <- ggplot(ff1, aes(y=delta_max_patch,x=year,colour=region)) +  theme_bw() +
		geom_point(shape=19) + facet_wrap(~threshold) +
		ylab(expression(Delta~S[max]~~~(km^2))) + scale_colour_brewer(palette="Set1",name="Sub-Region",guide=F) +
		theme(axis.text.x = element_text(angle=90,hjust=0)) +
		stat_quantile(quantiles=c(.10,.50,.90),linetype=2)

	print(g)
}

#
#' Call python routine to estimate distributions and likelihood ratio tests
#'
#' @param binDir folder with bin files with patch sizes
#'
#' @return data frame with results
#'
call_python_powlawfit <- function(binDir,fit=TRUE){

	ps <-  paste0("python ", oldcd,"/Code/powlawfit.py 1")

	setwd(binDir)
	if(fit){
		file.remove("fittedDistributions.txt")

		system(ps)
	}
	pyfit <- read.table("fittedDistributions.txt", header=TRUE,sep="\t",stringsAsFactors=FALSE)
	#names(pyfit)

	pyfit1 <- pyfit %>% group_by(model_name,file_name) %>% do(  {
		ss <- strsplit(.$file_name,"_")
		region <- ss[[1]][2]
		subregion <- ss[[1]][3]
		threshold <- ss[[1]][4]
		year <- gsub(".*\\.A([0-9]{4}).*","\\1",.$file_name)
		data.frame(region=region,subregion=subregion,threshold=threshold,year=year)})

	pyfit <- pyfit %>% inner_join(pyfit1) %>% arrange(region,subregion,threshold)

	# Lognormal without restriction has mu negative which results in a 99.7% probability is outside the range of patch data
	#

	lNorm <- pyfit  %>% ungroup() %>% filter(grepl("Log.*",model_name)) %>% select(model_name:xmin) %>% mutate(mu=exp(par1),si=exp(par2),lowd=mu/si^3,hid=mu*si^3)

	print(lNorm %>% mutate(higher= hid>.01) %>% group_by(model_name,higher) %>% summarize(n=n(), mlow=mean(lowd),mhi=mean(hid)) %>% mutate(modFreq= n/sum(n)))
	#
	#
	pyfit <- filter(pyfit, model_name!="LogNorm")

	pyfit <- pyfit %>% group_by(file_name) %>% mutate(delta_AICc= AICc - min(AICc),
													  norm_AICc_w = exp(-0.5*delta_AICc),
													  AICc_weight=exp(-0.5*delta_AICc)/sum(norm_AICc_w))

	setwd(oldcd)

	return(list(pyfit=pyfit,lNorm=lNorm))
	}

#
#' Call python routine to estimate distributions and likelihood ratio tests
#' for delta_Smax and delta_RSmax
#'
#' @param binDir folder with bin files with patch sizes
#'
#' @return data frame with results
#'
call_python_powlawfit_delta_threshold <- function(ff,binDir,fit=TRUE){

	setwd(binDir)

	group_by(ff, regsub,threshold) %>% do( nothing={
		zz <- file(paste0("delta_", .$regsub,"_",.$threshold,"_.bin")[1], "wb")
		writeBin(.$delta_max_patch,zz)
		close(zz)
		return(.$regsub)
	    })

	ps <-  paste0("python ", oldcd,"/Code/powlawfit.py 0")

	if(fit){
		file.remove("fittedDistributions.txt")

		system(ps)
	}
	pyfit <- read.table("fittedDistributions.txt", header=TRUE,sep="\t",stringsAsFactors=FALSE)
	#names(pyfit)

	pyfit1 <- pyfit %>% group_by(model_name,file_name) %>% do(  {
		ss <- strsplit(.$file_name,"_")
		regsub <- ss[[1]][2]
		threshold <- ss[[1]][3]
		data.frame(regsub=regsub,threshold=threshold)})

	pyfit <- pyfit %>% inner_join(pyfit1) %>% arrange(regsub,threshold)

	#
	#
	pyfit <- filter(pyfit, model_name!="LogNormPos")
	pyfit <- filter(pyfit, model_name!="PowerExp")

	pyfit <- pyfit %>% group_by(file_name) %>% mutate(delta_AICc= AICc - min(AICc),
													  norm_AICc_w = exp(-0.5*delta_AICc),
													  AICc_weight=exp(-0.5*delta_AICc)/sum(norm_AICc_w)) %>%
		ungroup() %>% select( -file_name )

	unlink("*.bin")
	setwd(oldcd)

	return(pyfit=pyfit)
	}


#Generacion de la Linea de Muerte para la UBA año 2020
#Solo necesita 32GB de memoria RAM  , 8 vCPU   y una hora para correr

#limpio la memoria
rm( list=ls() )  #remove all objects
gc()             #garbage collection

require("data.table")
require("lightgbm")
require("DiceKriging")
require("mlrMBO")

#en estos archivos queda el resultado
kbayesiana  <-  paste0("~/buckets/b2/opt_bayesiana_lgbm/linea_de_muerte.RDATA")

kBO_iter    <-  10  #cantidad de iteraciones de la Optimizacion Bayesiana

#------------------------------------------------------------------------------
#esta es la funcion de ganancia, que se busca optimizar
#se usa internamente a LightGBM
#se calcula internamente la mejor ganancia para todos los puntos de corte posibles

fganancia_logistic_lightgbm   <- function(probs, data)
{
  vlabels <- getinfo(data, "label")

  tbl <- as.data.table( list( "prob"=probs, "gan"= ifelse( vlabels==1, 29250, -750 ) ) )

  setorder( tbl, -prob )
  tbl[ , gan_acum :=  cumsum( gan ) ]
  gan <- max( tbl$gan_acum )

  return(  list( name= "ganancia", value=  gan, higher_better= TRUE ) )
}
#------------------------------------------------------------------------------
#funcion que va a optimizar la Bayesian Optimization

estimar_lightgbm <- function( x )
{
  modelo <-  lgb.train(data= dBO_train,
                       objective= "binary",  #la clase es binaria
                       eval= fganancia_logistic_lightgbm,  #esta es la fuciona optimizar
                       valids= list( valid1= dBO_test1 ),
                       first_metric_only= TRUE,
                       metric= "custom",  #ATENCION   tremendamente importante
                       num_iterations=  999999,  #un numero muy grande
                       early_stopping_rounds=  200,
                       min_data_in_leaf= as.integer( x$pmin_data_in_leaf ),
                       feature_fraction= 0.25,
                       learning_rate= 0.02,
                       feature_pre_filter= FALSE,
                       verbose= -1,
                       seed= 102191
                      )

  ganancia1  <- unlist(modelo$record_evals$valid1$ganancia$eval)[ modelo$best_iter ]

  #esta es la forma de devolver un parametro extra
  attr(ganancia1 ,"extras" ) <- list("pnum_iterations"= modelo$best_iter )

  cat( modelo$best_iter, ganancia1, "\n" )

  return( ganancia1 )
}
#------------------------------------------------------------------------------
#Aqui comienza el programa
dataset  <- fread("~/buckets/b1/datasets/fe_exthist.txt.gz")

campos_lags  <- setdiff(  colnames(dataset) ,  c("clase_ternaria","clase01", "numero_de_cliente","foto_mes") )

#agreglo los lags de orden 1
setorderv( dataset, c("numero_de_cliente","foto_mes") )
dataset[,  paste0( campos_lags, "_lag1") :=shift(.SD, 1, NA, "lag"), by=numero_de_cliente, .SDcols= campos_lags]

#agrego los deltas de los lags, de una forma nada elegante
for( vcol in campos_lags )
{
   dataset[,  paste0(vcol, "_delta1") := get( vcol)  - get(paste0( vcol, "_lag1"))]
}

#paso la clase a binaria que tome valores {0,1}  enteros
dataset[ , clase01 :=  ifelse( clase_ternaria=="BAJA+2", 1L, 0L)  ]

#los campos que se van a utilizar, intencionalmente no uso  numero_de_cliente
campos_buenos  <- setdiff(  colnames(dataset) ,  c("clase_ternaria","clase01","numero_de_cliente") )

#hago undersampling de los negativos
#me quedo con TODOS los positivos, pero con solo el 5% de los negativos
set.seed(102191)
dataset[ , azar:= runif( nrow(dataset) ) ]

dataset[ ( foto_mes>=201701 & foto_mes<=202003 & foto_mes!=201912 & ( clase01==1 | azar<=0.05) ),  BO_train := 1L]

#Testeo en 201902, el mismo mes pero un año antes
dataset[ foto_mes==201912,  BO_test1 := 1L]

#dejo los datos en el formato que necesita LightGBM
dBO_train  <- lgb.Dataset( data  = data.matrix(  dataset[ BO_train==1, campos_buenos, with=FALSE]),
                           label = dataset[ BO_train==1, clase01],
                           free_raw_data = TRUE
                         )

dBO_test1  <- lgb.Dataset( data  = data.matrix(  dataset[ BO_test1==1, campos_buenos, with=FALSE]),
                           label = dataset[ BO_test1==1, clase01],
                           free_raw_data = TRUE
                         )

dataset_aplicacion <- copy( dataset[ foto_mes==202005, ] )

#libero la memoria borrando el dataset
rm(dataset)
gc()

#Aqui comienza la configuracion de la Bayesian Optimization
configureMlr(show.learner.output = FALSE)

#configuro la busqueda bayesiana,  los hiperparametros que se van a optimizar
#por favor, no desesperarse por lo complejo
obj.fun <- makeSingleObjectiveFunction(
        name = "OptimBayesiana",  #un nombre que no tiene importancia
        fn   = estimar_lightgbm,  #aqui va la funcion que quiero optimizar
        minimize= FALSE,  #quiero maximizar la ganancia
        par.set = makeParamSet(
            makeNumericParam("pmin_data_in_leaf",  lower=  10, upper=  30000 )
            ),
        has.simple.signature = FALSE,  #porque le pase los parametros con makeParamSet
        noisy= TRUE
        )

ctrl  <-  makeMBOControl( save.on.disk.at.time = 600,  save.file.path = kbayesiana )
ctrl  <-  setMBOControlTermination(ctrl, iters = kBO_iter )
ctrl  <-  setMBOControlInfill(ctrl, crit = makeMBOInfillCritEI())

surr.km  <-  makeLearner("regr.km", predict.type= "se", covtype= "matern3_2", control = list(trace = FALSE))

if(!file.exists(kbayesiana))
{
  #lanzo la busqueda bayesiana
  run  <-  mbo(obj.fun, learner = surr.km, control = ctrl)
} else {
  #retoma el procesamiento en donde lo dejo
  run <- mboContinue( kbayesiana )
}

#En  run$x$pmin_data_in_leaf  ha quedo el optimo
#------------------------------------------------------------------------------

#calculo el modelo final
modelo_final <- lgb.train(data= dBO_train,
                          objective= "binary",
                          eval= fganancia_logistic_lightgbm,
                          valids= list( valid1= dBO_test1 ),
                          first_metric_only= TRUE,
                          metric= "custom",
                          num_iterations=  999999,
                          early_stopping_rounds=  200,
                          learning_rate= 0.02,  #ATENCION, este es el valor que se cambia
                          min_data_in_leaf= as.integer( run$x$pmin_data_in_leaf ),
                          feature_pre_filter= FALSE,
                          feature_fraction= 0.25,
                          verbose= -1,
                          seed= 102191
                         )

#Genero los archivos que voy a probar contra el Leaderboard Publico y quedarme con el mejor
prediccion_202005  <- predict( modelo_final,  data.matrix( dataset_aplicacion[  , campos_buenos, with=FALSE]))

#Genero  posibles probabilidades de corte, tener en
for( vprob_corte  in (25:25)/100 )  #de 0.15 a 0.35
{
  entrega  <- as.data.table( list( "numero_de_cliente"=  dataset_aplicacion[ , numero_de_cliente],
                                   "estimulo"=  (prediccion_202005> vprob_corte)  ) )

  #genero el archivo de salida
  fwrite( entrega, logical01=TRUE, sep=",",  file= paste0("~/buckets/b1/work/BORRADOR_lineademuerte_04_", vprob_corte*100, ".csv") )

  data = data.table('numero_de_cliente' = dataset_aplicacion[, numero_de_cliente], 'prob' = prediccion_202005)
  fwrite(data, sep = ',',  file = paste0("~/buckets/b1/work/BORRADOR_lineademuerte_04.probs"))
}

#Se deben probar con el metodo de busqueda binaria contra el leaderboard publico las salidas, y quedarse con el mejor