#' Model fitting
#'
#' @description This function fits the occupancy model of Diana et al. (2021).
#' Note that in the following the parameters are described with the notations used in the paper.
#'
#' @param data The data frame containing the data.
#' @param index_year The index of the column containing the year variable.
#' @param index_site The index of the column containing the site variable.
#' @param index_occ The index of the column containing the detections.
#' @param index_spatial_x The index of the x coordinate of the site.
#' @param index_spatial_y The index of the y coordinate of the site.
#' @param covariates_psi_text Indexes of the column of the occupancy probability. To be separated by a comma, eg. "5,6,8". Set to "0" if no covariate is available
#' @param covariates_p_text Indexes of the column of the detection probability.
#' @param prior_psi Prior mean for the occupancy probability
#' @param sigma_psi Standard deviation for the prior on the occupancy probability
#' @param prior_p Prior mean for the detection probability
#' @param sigma_p Standard deviation for the prior on the detection probability
#' @param usingYearDetProb Should the model include year-specific detection probabilities (as opposed to a constant one)?
#' @param usingSpatial Should the model include the auto-correlated spatial effects?
#' @param gridStep Step of the grid to use for the approximation of the auto-correlated spatial effects. Use \code{\link{buildSpatialGrid}} to show the grid for a value of \code{gridStep}.
#' @param storeRE Should the model store the site-specific independent random effects for each iteration (instead of just their mean across all chain)? Not suggested if the number of sites is greater than 1000.
#' @param nchain Number of chains.
#' @param nburn Number of burn-in iterations.
#' @param niter Number of (non burn-in) iterations.
#' @param verbose Should the progress of the MCMC be printed?.
#' @param computeGOF Should the model perform calculations of the goodness of fit?
#'
#' @importFrom magrittr %>%
#'
#' @export
#'
#' @return
#' A list with components:
#'
#' \itemize{
#'
#'  \item \code{modelResults} A list with components:
#'
#'       \describe{
#'
#'       \item{\code{beta_psi_output}}{ An array of dimensions \code{nchain} x \code{niter} x (number of coefficients for
#'       occupancy probability) containing the values of the coefficients of the occupancy probability.
#'      The coefficients are reported in the order (year r.e., space r.e., covariates for time-space interaction,
#'       standard covariates).}
#'
#'       \item{\code{eps_unique_output}}{ if \code{storeRE = T}, an array of dimensions
#'       \code{nchain} x \code{niter} x S containing the values of the site-specific independent random effects
#'        of the occupancy probability. If  \code{storeRE = F}, a matrix of dimensions \code{nchain} x S
#'        containing the mean value of the random effect across a chain.}
#'
#'       \item{\code{beta_p_output}}{ An array of dimensions \code{nchain} x \code{niter} x (number of coefficients for
#'       detection probability) containing the values of the coefficients of the detection probability.
#'        The coefficients are reported in the order (intercepts, covariates).}
#'
#'        \item{\code{sigma_T_output}}{ A array of dimensions \code{nchain} x \code{niter} containing the values of
#'        \eqn{\sigma_T}.}
#'
#'        \item{\code{l_T_output}}{ A array of dimensions \code{nchain} x \code{niter} containing the values of
#'        \eqn{l_T}.}
#'
#'        \item{\code{sigma_s_output}}{ A array of dimensions \code{nchain} x \code{niter} containing the values of
#'        \eqn{\sigma_S}.}
#'
#'        \item{\code{l_s_output}}{ A array of dimensions \code{nchain} x \code{niter} containing the values of
#'        \eqn{l_S}.}
#'
#'        \item{\code{sigma_eps_output}}{ A array of dimensions \code{nchain} x \code{niter} containing the values of
#'        \eqn{\sigma_\epsilon}.}
#'
#'        \item{\code{psi_mean_output}}{ A array of dimensions \code{nchain} x \code{niter} containing the values of
#'        the occupancy index.}
#'
#'        \item{\code{GOF_output}}{ A list with components:
#'
#'        \describe{
#'
#'        \item{  \code{gofYear_output} }{An array of dimensions \code{nchain} x \code{niter} x Y containing the values
#'          of the test statistics of yearly detections.}
#'
#'        \item{  \code{gofSpace_output} }{An array of dimensions \code{nchain} x \code{niter} x (M), where M is the number
#'        of regions in the approximation, containing the values of the test statistics of the detections in
#'        each region.}
#'
#'        \item{  \code{trueYearStatistics} }{ A vector containing the true values of the test statistics of the detections
#'         in each year.}
#'
#'        \item{  \code{trueSpaceStatistics} }{ A vector containing the true values of the test statistics of the detections
#'         in each region}
#'
#'        }
#'
#'        }
#'
#'      }
#'
#'
#'       \item \code{dataCharacteristics}: A list with components:
#'
#'       \describe{
#'
#'       \item{\code{Years}}{A vector with the years.}
#'
#'       \item{\code{X_tilde}}{A matrix of dimension M x 2, where M is the number of points chosen in the
#'       spatial approximation, with the location of the points used for the approximation. The points are
#'       arranged in the same as order as in the coefficients vector \code{beta_psi_output}.}
#'
#'       \item{\code{gridStep}}{The width of the grid chosen in the approximation.}
#'
#'       \item{\code{usingSptial}}{Same as in the input.}
#'
#'        \item{\code{usingYearDetProb}}{Same as in the input.}
#'
#'
#'       }
#'  }
#'
#' @examples
#'
#' modelResults <- runModel(sampleData,
#'                          index_year = 1,
#'                          index_site = 2,
#'                          index_occ = 8,
#'                          index_spatial_x = 3,
#'                          index_spatial_y = 4,
#'                          covariates_psi_text = "5",
#'                          covariates_p_text = "6-7",
#'                          usingSpatial = TRUE,
#'                          gridStep = .2,
#'                          nchain = 1,
#'                          nburn = 100,
#'                          niter = 100)
#'
runModel <- function(data, index_year, index_site,
                     index_occ, index_spatial_x = 0, index_spatial_y = 0,
                     covariates_psi_text, covariates_p_text,
                     prior_psi = .5, sigma_psi = 2,
                     prior_p = .5, sigma_p = 2,
                     usingYearDetProb = F,
                     usingSpatial = F, gridStep,
                     storeRE = F, nchain, nburn, niter,
                     verbose = T, computeGOF = T){

  print("Analyzing the data..")

  # CLEAN DATA
  {

    colnames(data)[c(index_year, index_site, index_occ)] <- c("Year","Site","Occ")

    if(usingSpatial){
      colnames(data)[c(index_spatial_x, index_spatial_y)] <- c("X_sp","Y_sp")
    }

    data$Year <- as.numeric(data$Year)

    S <- length(unique(data$Site))
    Y <- length(unique(data$Year))
    years <- sort(unique(data$Year))

    # transform site codes to numbers
    originalsites <- unique(data$Site)
    names(originalsites) <- seq_along(originalsites)
    data$Site <- as.numeric(names(originalsites)[match(data$Site, originalsites)])

    data <- data %>% dplyr::arrange(Site, Year)

    # define data for occupancies

    {
      k_s <- data %>% dplyr::group_by(Year, Site) %>%
        dplyr::summarise(Occupied = as.numeric(sum(Occ) > 0),
                         Visits = dplyr::n()) %>%
        dplyr::arrange(Site, Year)
      k_s <- as.data.frame(k_s)

      # match row in data_p to row in k_s
      indexes_occobs <- rep(1:nrow(k_s), k_s$Visits)

      Occs <- data$Occ
    }

    # covariate for psi

    {

      X_psi <- data[!duplicated(data[,c("Site","Year")]),] %>% dplyr::arrange(Site, Year)

      # year covariates
      {
        X_psi$Year <- as.factor(X_psi$Year)
        X_psi_year <- stats::model.matrix( ~ . - 1, data = X_psi[,c("Year"),drop = F])

        X_y_index <- apply(X_psi_year, 1, function(x) {which(x != 0)})
        X_y_index <- unlist(X_y_index)

      }

      # spatial covariates
      {

        if(usingSpatial){

          X_sp <- X_psi$X_sp
          Y_sp <- X_psi$Y_sp

          uniqueIndexesSite <- which(!duplicated(X_psi$Site))
          X_sp_unique <- X_sp[uniqueIndexesSite]
          Y_sp_unique <- Y_sp[uniqueIndexesSite]
          XY_sp_unique <- cbind(X_sp_unique, Y_sp_unique)

          # build the grid
          X_tilde <- as.matrix(buildGrid(XY_sp_unique, gridStep))
          XY_centers <- findClosestPoint(XY_sp_unique, X_tilde)

          # get rid of unused cells by refinding the grid
          X_tilde <- X_tilde[sort(unique(XY_centers)),]

          # refit
          XY_centers <- findClosestPoint(cbind(X_psi$X_sp,X_psi$Y_sp), X_tilde)
          XY_centers_unique <- findClosestPoint(XY_sp_unique, X_tilde)

          X_psi$Center <- as.factor(XY_centers)
          X_s_index <- XY_centers

          X_psi_s <- stats::model.matrix( ~ . - 1, data = X_psi[,c("Center"),drop = F])

          X_centers <- nrow(X_tilde)

        } else {
          X_centers <- 0
          XY_centers <- NULL
          X_psi_s <- NULL
          XY_sp_unique <- NULL
          X_s_index <- numeric(0)
          X_tilde <- NULL
          gridStep <- NULL
        }

      }

      # time space covariates
      {
        numTimeSpaceCov <- 0

        X_psi_yearcov <- scale(as.numeric(X_psi$Year))
        X_psi_yearcov_values <- sort(unique(X_psi_yearcov))

        if(usingSpatial){

          X_psi_spcov <- cbind(X_sp, Y_sp)

          X_timesp_cov <- cbind(X_psi_yearcov * X_sp, X_psi_yearcov * Y_sp)

          numTimeSpaceCov <- numTimeSpaceCov + 2

        } else {
          X_timesp_cov <- NULL
        }

      }

      # standard covariates
      {

        column_covariate_psi <- ExtractCovariatesFromText(covariates_psi_text)

        nameVariables_psi <- colnames(data)[column_covariate_psi]
        ncov_psi <- length(nameVariables_psi)

        if(ncov_psi > 0){

          X_psi_cov <- stats::model.matrix(~ ., data = X_psi[,column_covariate_psi,drop = F])[,-1]

        }

      }

      X_psi <- cbind(X_psi_year)
      if(usingSpatial) X_psi <- cbind(X_psi, X_psi_s, X_timesp_cov)
      if(ncov_psi > 0) X_psi <- cbind(X_psi, X_psi_cov)

    }

    # covariates for p

    {
      data_p <- data

      column_covariate_p <- ExtractCovariatesFromText(covariates_p_text)

      X_p_cov <- data_p[,column_covariate_p, drop = F]

      ncov_p <- ncol(X_p_cov)

      if(ncov_p > 0){
        nameVariables_p <- colnames(data_p)[column_covariate_p]
      } else {
        nameVariables_p <- c()
      }

      if(usingYearDetProb){

        data_p$Year <- as.factor(data_p$Year)
        X_p_year <- stats::model.matrix( ~ . - 1, data = data_p[,c("Year"),drop = F])

        X_y_index_p <- apply(X_p_year, 1, function(x) {which(x != 0)})
        X_y_index_p <- unlist(X_y_index_p)

        X_p <- X_p_year
      } else {
        X_p <- matrix(1, nrow = nrow(data_p), ncol = 1)
        X_y_index_p <- rep(1, nrow(data_p))
      }

      if(ncov_p > 0){
        X_p <- cbind(X_p, X_p_cov)
      }

      X_p <- as.matrix(X_p)

      p_intercepts <- ifelse(usingYearDetProb, Y, 1)

    }

    # data for GOF

    {

      k_s_all <- data %>% dplyr::arrange(Site, Year)

      trueYearStatistics <- k_s_all %>% dplyr::group_by(Year) %>%
        dplyr::summarise(Detections = sum(Occ)) %>% dplyr::arrange(Year)

      if(usingSpatial){

        k_s_all$SitePatch <- findClosestPoint(cbind(k_s_all$X_sp,k_s_all$Y_sp), X_tilde)

        trueSpaceStatistics <- k_s_all %>% dplyr::group_by(SitePatch) %>%
          dplyr::summarise(Detections = sum(Occ)) %>% dplyr::arrange(SitePatch)

        k_s_all <- k_s_all %>% dplyr::select(SitePatch, Year, ) %>% dplyr::mutate(Present = 0)

      } else {

        k_s_all <- k_s_all %>% dplyr::select(Year) %>% dplyr::mutate(Present = 0)

        trueSpaceStatistics <- NULL
      }

    }
  }

  # ASSIGN THE PRIOR
  {
    print("Setting up the priors")

    # fixed parameters
    {
      phi_psi <- 2
      phi_p <- 2

      a_l_T <- 1
      b_l_T <- 1

      a_sigma_T <- 2
      b_sigma_T <- .25

      a_l_s <- 1
      b_l_s <- 10

      a_sigma_s <- 2
      b_sigma_s <- 1

      a_sigma_eps <- 2
      b_sigma_eps <- .25

      sd_l_T <- .2
      sd_sigma_T <- .2
    }

    sigma_s <- b_sigma_s / (a_sigma_s - 1)
    sigma_T <- b_sigma_T / (a_sigma_T - 1)

    l_T <- a_l_T / b_l_T
    l_s <- a_l_s / b_l_s

    mu_psi <- invLogit(prior_psi)
    mu_p <- invLogit(prior_p)

    # priors on psi

    {

      b_psi <- c(rep(mu_psi,Y), rep(0, ncol(X_psi) - Y))
      B_psi <- matrix(0, nrow = ncol(X_psi), ncol = ncol(X_psi))

      C <- matrix(0, nrow = Y + X_centers, ncol = Y + X_centers)
      C[1:Y, 1:Y] <- K(1:Y,1:Y, sigma_T^2, l_T) + sigma_psi^2
      if(usingSpatial){
        C[Y + 1:X_centers, Y + 1:X_centers] <- K2(X_tilde,X_tilde, sigma_s^2, l_s)
      }

      B_psi[1:(Y + X_centers), 1:(Y + X_centers)] <- C

      B_psi[Y + X_centers + seq_len(numTimeSpaceCov),
            Y + X_centers + seq_len(numTimeSpaceCov)] <- diag(phi_psi^2, nrow = numTimeSpaceCov)

      if(ncov_psi > 0){

        C_covs <- diag(phi_psi^2, nrow = ncov_psi)

        B_psi[(Y + X_centers) + numTimeSpaceCov + 1:ncov_psi,
              (Y + X_centers) + numTimeSpaceCov + 1:ncov_psi] <- C_covs

      }

      inv_B_psi <- solve(B_psi)
    }

    # prior on p

    {
      b_p <- c(rep(mu_p, p_intercepts), rep(0, ncol(X_p) - p_intercepts))
      B_p <- matrix(0, nrow = ncol(X_p), ncol = ncol(X_p))
      diag(B_p)[1:p_intercepts] <- sigma_p^2

      if(ncov_p > 0){

        C <- diag(phi_p^2, nrow = ncov_p)

        B_p[p_intercepts + 1:ncov_p, p_intercepts + 1:ncov_p] <- C

      }

      inv_B_p <- solve(B_p)

    }
  }

  # run MCMC
  {
    # initialize output
    {
      beta_psi_output <- array(NA, dim = c(nchain , niter, ncol(X_psi)),
                               dimnames = list(c(), c(), colnames(X_psi)))

      beta_p_output <- array(NA, dim = c(nchain , niter, ncol(X_p)),
                             dimnames = list(c(), c(), colnames(X_p)))

      l_T_output <- matrix(NA, nrow = nchain, ncol = niter)
      l_s_output <- matrix(NA, nrow = nchain, ncol = niter)
      sigma_T_output <- array(NA, dim = c(nchain, niter))
      sigma_s_output <- array(NA, dim = c(nchain, niter))
      sigma_eps_output <- array(NA, dim = c(nchain, niter))

      psi_mean_output <- array(NA, dim = c(nchain, niter, Y))
      indexUniqueSite <- which(!duplicated(k_s$Site))
      namesUniqueSite <- k_s$Site[indexUniqueSite]

      gofYear_output <- array(NA, dim = c(nchain, niter, Y))
      if(usingSpatial){
        gofSpace_output <- array(NA, dim = c(nchain, niter, nrow(X_tilde)))
      } else {
        gofSpace_output <- NULL
      }

      if(storeRE){
        eps_unique_output <- array(NA, dim = c(nchain, niter, S),
                                   dimnames = list(c(), c(), namesUniqueSite))
      } else {
        eps_unique_output <- matrix(0, nchain , S)
        colnames(eps_unique_output) <- namesUniqueSite
      }

    }

    for (chain in 1:nchain) {

      # initialize parameters
      {

        print("Initializing the parameters")

        beta_psi <- b_psi
        psi <- as.vector(logit(X_psi %*% beta_psi))

        Xbetapsi <- X_psi %*% beta_psi

        beta_p <- b_p
        p <- as.vector(logit(X_p %*% beta_p))

        Xbetap <- X_p %*% beta_p

        eps_s <- rep(0, nrow(k_s))

        z <- rep(NA, nrow(k_s))
        for (i in 1:nrow(k_s)) {
          if(k_s[i,1] == 1){
            z[i] <- 1
          } else {
            z[i] <- stats::rbinom(1, 1, psi[i])
          }
        }

        # presences across each sampling occasion
        z_all <- z[indexes_occobs]

        # hyperparameters
        {
          l_T <- a_l_T / b_l_T
          l_s <- a_l_s / b_l_s

          sigma_T <- b_sigma_T / (a_sigma_T - 1)
          sigma_s <- b_sigma_s / (a_sigma_s - 1)

          sigma_eps <- b_sigma_eps / (a_sigma_eps - 1)

          # precompute parameters to update l_s
          if(usingSpatial){

            length_grid_ls <- 20
            l_s_grid <- seq(0.01, 0.5, length.out = length_grid_ls)
            K_s_grid <- array(NA, dim = c(X_centers, X_centers, length_grid_ls))
            inv_K_s_grid <- array(NA, dim = c(X_centers, X_centers, length_grid_ls))
            diag_K_s_grid <- array(NA, dim = c(X_centers, length_grid_ls))
            inv_chol_K_s_grid <- array(NA, dim = c(X_centers, X_centers, length_grid_ls))

            for (j in 1:length_grid_ls) {
              l_s <- l_s_grid[j]
              K_s_grid[,,j] <- K_l <- K2(X_tilde, X_tilde, 1, l_s) + diag(exp(-10), nrow = nrow(X_tilde))
              diag_K_s_grid[,j] <- FastGP::rcppeigen_get_diag(K_s_grid[,,j])
              inv_K_s_grid[,,j] <- FastGP::rcppeigen_invert_matrix(K_s_grid[,,j])
              inv_chol_K_s_grid[,,j] <- FastGP::rcppeigen_invert_matrix(FastGP::rcppeigen_get_chol(K_s_grid[,,j]))
            }

            l_s <- l_s_grid[length_grid_ls / 2]

          } else {
            l_s_grid <- NULL
            K_s_grid <- NULL
            inv_K_s_grid <- NULL
            diag_K_s_grid <- NULL
            inv_chol_K_s_grid <- NULL
          }

        }

      }

      for (iter in seq_len(nburn + niter)) {

        if(verbose){
          if(iter <= nburn){
            print(paste0("Chain = ",chain," - Burn-in Iteration = ",iter))
          } else {
            print(paste0("Chain = ",chain," - Iteration = ",iter - nburn))
          }
        }
        # print(paste0("l_s = ",l_s," / sigma_s = ",sigma_s,
        #              " / max(beta_psi) = ",max(abs(beta_psi[Y  + 1:X_centers]))))

        # sample z ----

        z <- sample_z_cpp(psi, p, as.matrix(k_s[,3:4]))
        z_all <- z[indexes_occobs]

        # sample psi ----

        list_psi <- update_psi(beta_psi, X_psi, Xbetapsi,
                               b_psi, inv_B_psi,
                               z, k_s, sites, Y, X_centers, ncov_psi,
                               X_y_index, X_s_index, numTimeSpaceCov,
                               usingSpatial, eps_s, sigma_eps)
        psi <- list_psi$psi
        beta_psi <- list_psi$beta
        Omega <- list_psi$Omega
        eps_s <- list_psi$eps_s
        XbetaY <- list_psi$XbetaY
        Xbetas <- list_psi$Xbetas
        Xbeta_cov <- list_psi$Xbeta_cov
        Xbetapsi <- list_psi$Xbeta

        # sample hyperparameters ---------

        list_hyperparameters <- update_hyperparameters(l_T, a_l_T, b_l_T, sd_l_T, sd_sigma_T,
                                                       sigma_T, a_sigma_T, b_sigma_T, Y,
                                                       beta_psi, inv_B_psi,
                                                       b_psi, sigma_psi,
                                                       l_s_grid, K_s_grid, inv_K_s_grid,
                                                       inv_chol_K_s_grid, diag_K_s_grid,
                                                       a_l_s, b_l_s,
                                                       sigma_s, a_sigma_s, b_sigma_s, X_tilde,
                                                       a_sigma_eps, b_sigma_eps,
                                                       usingSpatial,
                                                       XbetaY, Xbetas, Xbeta_cov,
                                                       eps_s, k_s,
                                                       z, X_psi, Omega, X_y_index)
        l_T <- list_hyperparameters$l_T
        sigma_T <- list_hyperparameters$sigma_T
        sigma_s <- list_hyperparameters$sigma_s
        l_s <- list_hyperparameters$l_s
        index_l_s <- list_hyperparameters$index_l_s
        inv_B_psi <- list_hyperparameters$inv_B_psi
        sigma_eps <- list_hyperparameters$sigma_eps

        # sample p ----------------

        if(sum(z) > 0){
          list_p <- update_p(beta_p, Occs, z_all, X_p, b_p, inv_B_p, Xbetap,
                             ncov_p, p_intercepts, X_y_index_p, usingYearDetProb)
          p <- list_p$p
          beta_p <- list_p$beta
          Xbetap <- list_p$Xbeta
        }

        # write parameters  -------------------------------------------------------

        if(iter > nburn){

          beta_psi_output[chain,iter - nburn,] <- beta_psi

          eps_unique <- eps_s[indexUniqueSite]

          if(usingSpatial){
            a_s_unique <- beta_psi[Y + XY_centers_unique] +
              eps_unique
            psi_mean_output[chain,iter - nburn,] <- computeYearEffect(Y, a_s_unique, beta_psi)
          } else {
            psi_mean_output[chain,iter - nburn,] <- logit(beta_psi[1:Y]) + mean(eps_unique)
          }

          if(storeRE){
            eps_unique_output[chain, iter - nburn,] <- eps_s[indexUniqueSite]
          } else {
            eps_unique_output[chain,] <- eps_unique_output[chain,] +
              (1 / niter)  * eps_s[indexUniqueSite]
          }

          sigma_eps_output[chain, iter - nburn] <- sigma_eps

          l_T_output[chain, iter - nburn] <- l_T
          sigma_T_output[chain, iter - nburn] <- sigma_T

          l_s_output[chain, iter - nburn] <- l_s
          sigma_s_output[chain, iter - nburn] <- sigma_s

          beta_p_output[chain,iter - nburn,] <- beta_p

          # GOF
          if (computeGOF) {
            k_s_all$Present <- simulateDetections(p, z_all)

            detectionsByYear <- k_s_all %>% dplyr::group_by(Year) %>%
              dplyr::summarise(Detections = sum(Present))

            gofYear_output[chain,iter - nburn,] <- detectionsByYear$Detections

            if(usingSpatial){

              detectionsByPatch <- k_s_all %>% dplyr::group_by(SitePatch) %>%
                dplyr::summarise(Detections = sum(Present))

              gofSpace_output[chain,iter - nburn,] <- detectionsByPatch$Detections

            }

          }

        }

      }

    }

  }

  # store summaries of the dataset
  {

    dataCharacteristics <- list("Years" = years,
                                "X_tilde" = X_tilde,
                                "originalsites" = originalsites,
                                "gridStep" = gridStep,
                                "XY_sp_unique" = XY_sp_unique,
                                "usingYearDetProb" = usingYearDetProb,
                                "usingSpatial" = usingSpatial,
                                "numTimeSpaceCov" = numTimeSpaceCov,
                                "X_psi_yearcov_values" = X_psi_yearcov_values,
                                "nameVariables_psi" = nameVariables_psi,
                                "nameVariables_p" = nameVariables_p)
  }

  # create model output
  {
    GOF_output <- list(
      "trueYearStatistics" = trueYearStatistics,
      "trueSpaceStatistics" = trueSpaceStatistics,
      "gofYear_output" = gofYear_output,
      "gofSpace_output" = gofSpace_output
    )

    modelOutput <- list(
      "beta_psi_output" = beta_psi_output,
      "eps_unique_output" = eps_unique_output,
      "beta_p_output" = beta_p_output,
      "sigma_T_output" = sigma_T_output,
      "l_T_output" = l_T_output,
      "sigma_s_output" = sigma_s_output,
      "l_s_output" = l_s_output,
      "sigma_eps_output" = sigma_eps_output,
      "psi_mean_output" = psi_mean_output,
      "GOF_output" = GOF_output
    )
  }

  list("dataCharacteristics" = dataCharacteristics,
       "modelOutput" = modelOutput)
}


###############################################################################
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
###############################################################################
# Examples for the R/Finance Presentation
# Copyright (C) 2012  Michael Kapler
#
# For more information please visit my blog at www.SystematicInvestor.wordpress.com
# or drop me a line at TheSystematicInvestor at gmail
###############################################################################

#############################################################################
# Seasonality Analysis - TIME patterns
# R/Finance 2012
###############################################################################
seasonality.test <- function()
{
	#*****************************************************************
	# Load historical data
	#******************************************************************
	load.packages('quantmod')
	ticker = 'WMT'

	data = getSymbols(ticker, src = 'yahoo', from = '1970-01-01', auto.assign = F)
		data = adjustOHLC(data, use.Adjusted=T)

	data = data['1980::2012:04:07']

	#*****************************************************************
	# Look at the Month of the Year Seasonality
	#******************************************************************
png(filename = 'plot.month.year.seasonality.png', width = 600, height = 500, units = 'px', pointsize = 12, bg = 'white')
	month.year.seasonality(data, ticker)
dev.off()

	#*****************************************************************
	# Look at What seasonally happens in the first 20 days of May
	#******************************************************************
	# Find first day of May: it is one day after the last day of April
	month.ends = endpoints(data, 'months')
		month.ends = month.ends[month.ends > 0 & month.ends < nrow(data)]
		index = which(format(index(data), '%b')[month.ends] == 'Apr')

png(filename = 'plot.time.seasonality.png', width = 600, height = 500, units = 'px', pointsize = 12, bg = 'white')
	layout(1)
	time.seasonality(data, 1 + month.ends[index], 20, ticker)
dev.off()
}

#############################################################################
# Pattern Matching - PRICE patterns
# R/Finance 2012
###############################################################################
pattern.test <- function()
{
	#*****************************************************************
	# Load historical data
	#******************************************************************
	load.packages('quantmod')
	ticker = 'SPY'

	data = getSymbols(ticker, src = 'yahoo', from = '1970-01-01', auto.assign = F)
		data = adjustOHLC(data, use.Adjusted=T)

	data = data['::2012:04:07']
	#*****************************************************************
	# Find historical Matches similar to the last 90 days of price history
	#******************************************************************
png(filename = 'plot1.time.series.pattern.matching.png', width = 600, height = 500, units = 'px', pointsize = 12, bg = 'white')
	matches = bt.matching.find(Cl(data), main = ticker,	n.query=90, plot=TRUE)
dev.off()

png(filename = 'plot2.time.series.pattern.matching.png', width = 600, height = 500, units = 'px', pointsize = 12, bg = 'white')
	out = bt.matching.overlay(matches, plot=TRUE)
dev.off()

	#*****************************************************************
	# Find Classical Techical Patterns, based on
	# Pattern Matching. Based on Foundations of Technical Analysis
	# by A.W. LO, H. MAMAYSKY, J. WANG
	#******************************************************************
png(filename = 'plot.pattern.matching.png', width = 600, height = 500, units = 'px', pointsize = 12, bg = 'white')
	plot.patterns(data, 190, ticker)
dev.off()
}

#############################################################################
# Month of the Year Seasonality
#' @export
#############################################################################
month.year.seasonality <- function
(
	data,	# xts time series data
	ticker,
	lookback.len = 20*252	# last 20 years of data
)
{
	data = last(data, lookback.len)
		nperiods = nrow(data)

	#*****************************************************************
	# Compute monthly returns
	#******************************************************************
	# find month ends
	month.ends = endpoints(data, 'months')
		month.ends = month.ends[month.ends > 0]

	prices = Cl(data)[month.ends]
	ret = prices / mlag(prices) - 1
		ret = ret[-1]

	ret.by.month = create.monthly.table(ret)

	#*****************************************************************
	# Plot
	#******************************************************************
	data_list = lapply(apply(ret.by.month, 2, list), '[[', 1)
	group.seasonality(data_list, paste(ticker, 'Monthly', join(format(range(index(data)), '%d-%b-%Y'), ' to\n')))

}

#' @export
group.seasonality <- function
(
	data_list,	# data list for each group
	smain,
	...
)
{
	#*****************************************************************
	# Compute group stats
	#******************************************************************
	out = compute.stats( data_list,
		list(Sharpe=function(x) mean(x,na.rm=T)/sd(x,na.rm=T),
			'% Positive'=function(x) sum(x > 0,na.rm=T)/sum(!is.na(x)),
			Min=function(x) min(x,na.rm=T),
			Max=function(x) max(x,na.rm=T),
			Avg=function(x) mean(x,na.rm=T),
			Med=function(x) median(x,na.rm=T),
			StDev=function(x) sd(x,na.rm=T)
			)
		)

	#*****************************************************************
	# Plot
	#******************************************************************
	layout(mat=matrix(1:4, 2, 2, byrow=FALSE))
	par(mar=c(4, 3, 2, 2))
	col = spl('lightgray,red')

	stats.names = spl('Sharpe,% Positive,Min,Avg')
	for(i in stats.names) {
		barplot(100*out[i,], names.arg = colnames(out),
			col=iif(out[i,] > 0, col[1], col[2]),
			main=iif(i == stats.names[1], paste(smain,' ', i, sep=''), i),
			border = 'darkgray',las=2, ...)
		grid(NA,NULL)
		abline(h=0, col='black')
	}
}

#############################################################################
# Time seasonally for given periods
#' @export
###############################################################################
time.seasonality <- function
(
	data,			# xts time series data
	period.starts,	# locations
	period.len,		# number of trading days to examine
	ticker
)
{
	nperiods = len(period.starts)
	dates = index(data)
		ndates = len(dates)

	#*****************************************************************
	# Compute returns, construct trading.days matrix
	#******************************************************************
	prices = Cl(data)
	ret = prices / mlag(prices) - 1
		ret = c( as.double(ret), rep(NA, period.len))

	# 1:period.len by period.starts matrix
	trading.days = sapply(period.starts, function(i) ret[i : (i + period.len - 1)])

	#*****************************************************************
	# Compute stats
	#******************************************************************
	# exclude periods that have insufficient data
	periods.index = 1:nperiods
		temp = count(trading.days)
	periods.index = periods.index[temp > 0.9 * median(temp)]

	# last period
	last.period = trading.days[, nperiods]
		last.period = 100 * ( cumprod(1 + last.period) - 1 )

	# average period
	avg.period = apply(trading.days[, periods.index], 1, mean, na.rm=T)
		avg.period = 100 * ( cumprod(1 + avg.period) - 1 )

	# ranges
	temp = 100*(apply(1 + trading.days[, periods.index], 2, cumprod) - 1)
	quantiles = apply(temp, 1, quantile, probs = c(75, 25)/100, na.rm=T)

	#*****************************************************************
	# Create Plot
	#******************************************************************
	cols = spl('blue,red,gray')

	par(mar=c(4,4,2,1))
	plot(avg.period, type='n', xaxt = 'n', xlim=c(1,period.len),
		ylim=range(avg.period, last.period, quantiles, na.rm=T),
		main = ticker, xlab = 'Trading Days', ylab = 'Avg % Profit/Loss')

	grid()
	axis(1, 1:period.len)

	lines(quantiles[1,], type='l', lwd=2, col=cols[3])
	lines(quantiles[2,], type='l', lwd=2, col=cols[3])

	lines(last.period, type='b', lwd=2, col=cols[2], bg=cols[2], pch=24)
	lines(avg.period, type='b', lwd=2, col=cols[1], bg=cols[1], pch=22)

	first.year = format(dates[period.starts][periods.index[1]], '%Y')
	last.year = format(dates[period.starts][last(periods.index)], '%Y')
	last.period.start.date = format(dates[period.starts[nperiods]], '%d %b %Y')
	last.period.end.date = format(dates[ndates], '%d %b %Y')

	if( (period.starts[nperiods] + period.len - 1) < ndates ) {
		last.period.end.date = format(dates[period.starts[nperiods] + period.len - 1], '%d %b %Y')
	}

	plota.legend(c(paste('Avgerage for', first.year, '-', last.year),
			paste(last.period.start.date, '-', last.period.end.date),
			'Top 25% / Bot 25%'), cols)
}

#############################################################################
# Find and Plot Classical Techical Patterns
#' @export
#############################################################################
plot.patterns <- function
(
	data,	# xts time series data
	n,		# lookback period to search for the patterns
	ticker,
	patterns = pattern.db()	# database with patterns
)
{
	#*****************************************************************
	# Find Extrema
	#******************************************************************
	load.packages('sm')

	sample = last(data, n)

	obj = find.extrema( Cl(sample) )
		mhat = obj$mhat
		mhat.extrema.loc = obj$mhat.extrema.loc
		data.extrema.loc = obj$data.extrema.loc
		n.index = len(data.extrema.loc)

	#*****************************************************************
	# Plot
	#******************************************************************
		plota.control$col.border = 'gray'
	plota(sample, type='hl',col='gray')
		plota.lines(mhat, col='magenta', lwd=2)
		plota.lines(sample, col='blue')

	if(n.index > 0) {
		plota.lines(sample[data.extrema.loc], type='p', col='blue', lwd=3, pch=19)
		out = find.patterns(obj, patterns = patterns, silent=F, plot=T)
	}

	plota.legend(c(paste(ticker, join(format(range(index(sample)), '%d%b%Y'), ' - ')),
				'Close,Kernel,Pattern(s)'),
				'gray,blue,magenta,orange')

}

#############################################################################
# Find maxima and minima using Kernel estimate
#' @export
#############################################################################
find.extrema <- function(
	x	# time series
)
{
	if(is.xts(x)) {
		y = as.vector( Cl(x) )
	} else {
		y = x
	}
	n = len(y)
	t = 1:n

	# Fit kernel
	# stat.epfl.ch/files/content/sites/stat/files/users/MdC/notes_3.pdf
	h = h.select(t, y, method = 'cv')
		temp = sm.regression(t, y, h=h, display = 'none')
	mhat = approx(temp$eval.points, temp$estimate, t, method='linear')$y

	# page 15
	# find locations of local maxima and minima in mhat
	temp = diff(sign(diff(mhat)))
	loc = which( temp != 0 ) + 1
		loc.dir = -sign(temp[(loc - 1)])

	# check
	# temp = cbind(mhat[(loc - 1)],mhat[(loc)],mhat[(loc + 1)])
	# cbind(round(temp,2), loc.dir, apply(temp, 1, which.max), apply(temp, 1, which.min))

	# page 16
	# find locations of local maxima and minima in original data with in +1/-1 local maxima and minima in mhat
	temp = c( y[1], y, y[n] )
	temp = cbind(temp[loc], temp[(loc + 1)], temp[(loc + 2)])
		max.index = loc + apply(temp, 1, which.max) - 2
		min.index = loc + apply(temp, 1, which.min) - 2
	data.loc = iif(loc.dir > 0, max.index, min.index)
	data.loc = iif(data.loc < 1, 1, iif(data.loc > n, n, data.loc))

	if(is.xts(x)) mhat = make.xts(mhat, index(x))

	return(list(data = y, mhat = mhat, extrema.dir = loc.dir,
		mhat.extrema.loc = loc, data.extrema.loc = data.loc))
}

#############################################################################
# Find Classical Techincal patterns
#' @export
#############################################################################
find.patterns <- function
(
	obj, 	# extrema points
	patterns = pattern.db(),
	silent=T,
	plot=T
)
{
	data = obj$data
	mhat = obj$mhat
	extrema.dir = obj$extrema.dir
	data.extrema.loc = obj$data.extrema.loc
	n.index = len(data.extrema.loc)

	if(is.xts(mhat)) {
		dates = index4xts(obj$mhat)
	} else {
		dates = 1:len(data)
	}

	# Semi-transparent orange color
	col = col.add.alpha('orange', alpha=150)

	out = out.rownames = c()

	# search for patterns
	for(i in 1:n.index) {

		for(pattern in patterns) {

			# check same sign
			if( pattern$start * extrema.dir[i] > 0 ) {

				# check that there is suffcient number of extrema to complete pattern
				if( i + pattern$len - 1 <= n.index ) {

					# create enviroment to check pattern: E1,E2,...,En; t1,t2,...,tn
					envir.data = c(data[data.extrema.loc][i:(i + pattern$len - 1)],
									data.extrema.loc[i:(i + pattern$len - 1)])
						names(envir.data) = c(paste('E', 1:pattern$len, sep=''),
												paste('t', 1:pattern$len, sep=''))
					envir.data = as.list(envir.data)

					# double top/bottom patterns require all extrema [we will exclude the first point(E1/t1)]
					#envir.data$E = data[data.extrema.loc][i:n.index]
					envir.data$E = data[data.extrema.loc][-c(1:i)]
					envir.data$t = data.extrema.loc[-c(1:i)]

					# check if pattern was found
					if( eval(pattern$formula, envir = envir.data) ) {
						if(!silent) cat('Found', pattern$name, 'at', i, '\n')

						if(plot & !is.null(pattern$plot)) {
							temp = dates[data.extrema.loc[i:(i + pattern$len - 1)]]
								names(temp) = paste('d', 1:pattern$len, sep='')
							envir.data = c( envir.data, temp )
							envir.data$d = dates[data.extrema.loc[-c(1:i)]]
							envir.data$col = col

							eval(pattern$plot, envir = envir.data)
						}

						# record
						out.rownames = c(out.rownames, pattern$name)
						out = rbind(out, c(data.extrema.loc[i],
											iif(is.null(pattern$last.point),
												data.extrema.loc[(i + pattern$len - 1)],
												eval(pattern$last.point, envir = envir.data)
												)))
					}
				}
			}
		}
	}

	if(len(out)>0) {
		colnames(out) = spl('start,end')
		rownames(out) = out.rownames
	}
	return(out)
}

#############################################################################
# Find historical patterns in the data over a rolling window
#' @export
#############################################################################
find.all.patterns.window <- function()
{
	#*****************************************************************
	# Load historical data
	#******************************************************************
	load.packages('quantmod')
	ticker = 'SPY'

	data = getSymbols(ticker, src = 'yahoo', from = '1970-01-01', auto.assign = F)
		data = adjustOHLC(data, use.Adjusted=T)

	data = data['2010::']
	#*****************************************************************
	# Search for all patterns over a rolling window
	#******************************************************************
	load.packages('sm')
	history = as.vector(coredata(Cl(data)))
	window.len = 90
	patterns = pattern.db()

	found.patterns = c()

	for(t in window.len : (len(history)-1)) {
		sample = history[(t - window.len + 1):t]
		obj = find.extrema( sample )

		if(len(obj$data.extrema.loc) > 0) {
			out =  find.patterns(obj, patterns = patterns, silent=F, plot=F)

			if(len(out)>0) found.patterns = rbind(found.patterns,cbind(t,out,t-window.len+out))
		}
		if( t %% 10 == 0) cat(t, 'out of', len(history), '\n')
	}
	colnames(found.patterns) = spl('t,start,end,tstart,tend')

	#*****************************************************************
	# Examine found patterns
	#******************************************************************
	# check what patterns are not found
	setdiff(unique(names(patterns)), unique(rownames(found.patterns)))

	# number of matches for each pattern
	frequency = tapply(rep(1,nrow(found.patterns)), rownames(found.patterns), sum)
	barplot(frequency)

	# determine starting Time for a pattern
	index = which(rownames(found.patterns)=='HS')
	found.patterns[ index[1:10],]

	# plot
	pattern.index = index[1]
	t = found.patterns[pattern.index, 't'];
	plot.patterns(data[1:t,], window.len, ticker, patterns)

	# plot start/end of this pattern
	sample = data[(t - window.len + 1):t]
		start.pattern = found.patterns[pattern.index, 'start']
		end.pattern = found.patterns[pattern.index, 'end']
	abline(v = index(sample)[start.pattern]);
	abline(v = index(sample)[end.pattern]);

	index(sample)[start.pattern]
	index(data[(t - window.len + start.pattern),])

}

#############################################################################
# Compute conditional returns for each pattern over a rolling window
#############################################################################
bt.patterns.test <- function()
{
	#*****************************************************************
	# Load historical data
	#******************************************************************
	load.packages('quantmod')
	ticker = 'SPY'

	data = getSymbols(ticker, src = 'yahoo', from = '1970-01-01', auto.assign = F)
		data = adjustOHLC(data, use.Adjusted=T)

	#data = data['2010::']

	#*****************************************************************
	# Setup
	#******************************************************************
	# page 14-15, L = 35 and d = 3, rolling the length of the window at L + d,
	# The parameter d controls for the fact that in practice we do not observe a
	# realization of a given pattern as soon as it has completed. Instead, we assume
	# that there may be a lag between the pattern completion and the time
	# of pattern detection. To account for this lag, we require that the final extremum
	# that completes a pattern occurs on day t + L - 1; hence d is the number
	# of days following the completion of a pattern that must pass before the pattern
	# is detected.
	#In particular, we compute postpattern returns starting from the end of trading
	#day t + L + d, that is, one day after the pattern has completed. For
	#example, if we determine that a head-and-shoulder pattern has completed
	# on day t + L - 1 (having used prices from time t through time t + L + d - 1),
	#we compute the conditional one-day gross return as Z1=Yt+L+d+1/Yt+L+d.

	#*****************************************************************
	# Search for all patterns over a rolling window
	#******************************************************************
	load.packages('sm')
	history = as.vector(coredata(Cl(data)))

	window.L = 35
	window.d = 3
	window.len = window.L + window.d

	patterns = pattern.db()

	found.patterns = c()

	for(t in window.len : (len(history)-1)) {
		ret = history[(t+1)]/history[t]-1

		sample = history[(t - window.len + 1):t]
		obj = find.extrema( sample )

		if(len(obj$data.extrema.loc) > 0) {
			out =  find.patterns(obj, patterns = patterns, silent=F, plot=F)

			if(len(out)>0) found.patterns = rbind(found.patterns,cbind(t,out,t-window.len+out, ret))
		}
		if( t %% 10 == 0) cat(t, 'out of', len(history), '\n')
	}
	colnames(found.patterns) = spl('t,start,end,tstart,tend,ret')

	#*****************************************************************
	# Clean found patterns
	#******************************************************************
	# remove patterns that finished after window.L
	found.patterns = found.patterns[found.patterns[,'end'] <= window.L,]

	# remove the patterns found multiple times, only keep first one
	pattern.names = unique(rownames(found.patterns))
	all.patterns = c()
	for(name in pattern.names) {
		index = which(rownames(found.patterns) == name)
		temp = NA * found.patterns[index,]

		i.count = 0
		i.start = 1
		while(i.start < len(index)) {
			i.count = i.count + 1
			temp[i.count,] = found.patterns[index[i.start],]
			subindex = which(found.patterns[index,'tstart'] > temp[i.count,'tend'])

			if(len(subindex) > 0) {
				i.start = subindex[1]
			} else break
		}
		all.patterns = rbind(all.patterns, temp[1:i.count,])
	}

	#*****************************************************************
	# Plot
	#******************************************************************
png(filename = 'plot1.png', width = 600, height = 500, units = 'px', pointsize = 12, bg = 'white')
	# number of matches for each pattern
	frequency = tapply(rep(1,nrow(all.patterns)), rownames(all.patterns), sum)
	layout(1)
	barplot.with.labels(frequency/100, 'Frequency for each Pattern')
dev.off()

png(filename = 'plot2.png', width = 600, height = 500, units = 'px', pointsize = 12, bg = 'white')
	# pattern seasonality
	all.patterns[,'ret'] = history[(all.patterns[,'t']+20)] / history[all.patterns[,'t']] - 1
	data_list = tapply(all.patterns[,'ret'], rownames(all.patterns), list)
	group.seasonality(data_list, '20 days after Pattern')
dev.off()

png(filename = 'plot3.png', width = 600, height = 500, units = 'px', pointsize = 12, bg = 'white')
	# time pattern seasonality
	layout(1)
	name = 'BBOT'
	index = which(rownames(all.patterns) == name)
	time.seasonality(data, all.patterns[index,'t'], 20, name)
dev.off()

	#t.test(out[,'ret'])$p.value
	#tapply(out[,'ret'], rownames(out), function(x) t.test(x)$p.value)

}

###############################################################################
# Pattern Matching
#
# Based on Foundations of Technical Analysis: Computational Algorithms, Statistical Inference, and Empirical Implementation
# by A.W. LO, H. MAMAYSKY, J. WANG
###############################################################################
# http://thepatternsite.com/
#' @export
###############################################################################
pattern.db <- function()
{
	# page 12
	patterns = list()

	#*****************************************************************
	# You can reference E1,E2,...,En and t1,t2,...,tn in pattern formula
	#******************************************************************
	# Head-and-shoulders (HS)
	#******************************************************************
	pattern = list()
	pattern$len = 5
	pattern$start = 'max'
	pattern$formula = expression(
		# E3 > E1, E3 > E5
		E3 > E1 &
		E3 > E5 &

		# E1 and E5 are within 1.5 percent of their average
		abs(E1 - (E1+E5)/2) < 1.5/100 * (E1+E5)/2 &
		abs(E5 - (E1+E5)/2) < 1.5/100 * (E1+E5)/2 &

		# E2 and E4 are within 1.5 percent of their average
		abs(E2 - (E2+E4)/2) < 1.5/100 * (E2+E4)/2 &
		abs(E4 - (E2+E4)/2) < 1.5/100 * (E2+E4)/2
		)
	pattern$plot = expression({
		lines(c(d1,d2,d3,d4,d5), c(E1,E2,E3,E4,E5), lwd=10, col=col)
			text(d3, E3, 'HS', adj=c(0.5,-0.5), xpd=TRUE)
		})
	patterns$HS = pattern

	#*****************************************************************
	# Inverted Head-and-shoulders (IHS)
	#******************************************************************
	pattern = list()
	pattern$len = 5
	pattern$start = 'min'
	pattern$formula = expression(
		# E3 < E1, E3 < E5
		E3 < E1 &
		E3 < E5 &

		# E1 and E5 are within 1.5 percent of their average
		abs(E1 - (E1+E5)/2) < 1.5/100 * (E1+E5)/2 &
		abs(E5 - (E1+E5)/2) < 1.5/100 * (E1+E5)/2 &

		# E2 and E4 are within 1.5 percent of their average
		abs(E2 - (E2+E4)/2) < 1.5/100 * (E2+E4)/2 &
		abs(E4 - (E2+E4)/2) < 1.5/100 * (E2+E4)/2
		)
	pattern$plot = expression({
		lines(c(d1,d2,d3,d4,d5), c(E1,E2,E3,E4,E5), lwd=10, col=col)
			text(d3, E3, 'IHS', adj=c(0.5,1), xpd=TRUE)
		})
	patterns$IHS = pattern

	#*****************************************************************
	# Broadening tops (BTOP)
	#******************************************************************
	pattern = list()
	pattern$len = 5
	pattern$start = 'max'
	pattern$formula = expression(
		# E1 < E3 < E5
		E1 < E3 &
		E3 < E5 &
		E2 > E4
		)
	pattern$plot = expression({
		beta = ols(cbind(1,c(t1,t3,t5)),c(E1,E3,E5))$coefficients
		lines(c(d1,d3,d5), beta[1] + beta[2]*c(t1,t3,t5), lwd=10, col=col)
		lines(c(d2,d4), c(E2,E4), lwd=10, col=col)
			text(d3, min(E2,E4), 'BTOP', adj=c(0.5,1), xpd=TRUE)
		})
	patterns$BTOP = pattern

	#*****************************************************************
	# Broadening bottoms (BBOT)
	#******************************************************************
	pattern = list()
	pattern$len = 5
	pattern$start = 'min'
	pattern$formula = expression(
		# E1 > E3 > E5
		E1 > E3 &
		E3 > E5 &
		E2 < E4
		)
	pattern$plot = expression({
		beta = ols(cbind(1,c(t1,t3,t5)),c(E1,E3,E5))$coefficients
		lines(c(d1,d3,d5), beta[1] + beta[2]*c(t1,t3,t5), lwd=10, col=col)
		lines(c(d2,d4), c(E2,E4), lwd=10, col=col)
			text(d3, max(E2,E4), 'BBOT', adj=c(0.5,0), xpd=TRUE)
		})
	patterns$BBOT = pattern

	#*****************************************************************
	# Triangle tops (TTOP)
	#******************************************************************
	pattern = list()
	pattern$len = 5
	pattern$start = 'max'
	pattern$formula = expression(
		# E1 > E3 > E5
		E1 > E3 &
		E3 > E5 &
		E2 < E4
		)
	pattern$plot = expression({
		beta = ols(cbind(1,c(t1,t3,t5)),c(E1,E3,E5))$coefficients
		lines(c(d1,d3,d5), beta[1] + beta[2]*c(t1,t3,t5), lwd=10, col=col)
		lines(c(d2,d4), c(E2,E4), lwd=10, col=col)
			text(d3, min(E2,E4), 'TTOP', adj=c(0.5,1), xpd=TRUE)
		})
	patterns$TTOP = pattern

	#*****************************************************************
	# Triangle bottoms (TBOT)
	#******************************************************************
	pattern = list()
	pattern$len = 5
	pattern$start = 'min'
	pattern$formula = expression(
		# E1 < E3 < E5
		E1 < E3 &
		E3 < E5 &
		E2 > E4
		)
	pattern$plot = expression({
		beta = ols(cbind(1,c(t1,t3,t5)),c(E1,E3,E5))$coefficients
		lines(c(d1,d3,d5), beta[1] + beta[2]*c(t1,t3,t5), lwd=10, col=col)
		lines(c(d2,d4), c(E2,E4), lwd=10, col=col)
			text(d3, max(E2,E4), 'TBOT', adj=c(0.5,0), xpd=TRUE)
		})
	patterns$TBOT = pattern

	#*****************************************************************
	# Rectangle tops (RTOP)
	#******************************************************************
	pattern = list()
	pattern$len = 5
	pattern$start = 'max'
	pattern$formula = expression({
		avg.top = (E1+E3+E5)/3
		avg.bop = (E2+E4)/2

		# tops E1,E3,E5 are within 0.75 percent of their average
		abs(E1 - avg.top) < 0.75/100 * avg.top &
		abs(E3 - avg.top) < 0.75/100 * avg.top &
		abs(E5 - avg.top) < 0.75/100 * avg.top &

		# bottoms E2,E4 are within 0.75 percent of their average
		abs(E2 - avg.bop) < 0.75/100 * avg.bop &
		abs(E4 - avg.bop) < 0.75/100 * avg.bop &

		# lowest top > highest bottom
		min(E1,E3,E5) > max(E2,E4)
		})
	pattern$plot = expression({
		avg.top = (E1+E3+E5)/3
		avg.bop = (E2+E4)/2

		lines(c(d1,d3,d5), rep(avg.top,3), lwd=10, col=col)
		lines(c(d2,d4), rep(avg.bop,2), lwd=10, col=col)
			text(d3, min(E2,E4), 'RTOP', adj=c(0.5,-0.5), xpd=TRUE)
		})
	patterns$RTOP = pattern

	#*****************************************************************
	# Rectangle bottoms (RBOT)
	#******************************************************************
	pattern = list()
	pattern$len = 5
	pattern$start = 'min'
	pattern$formula = expression({
		avg.top = (E2+E4)/2
		avg.bop = (E1+E3+E5)/3

		# tops E2,E4 are within 0.75 percent of their average
		abs(E2 - avg.top) < 0.75/100 * avg.top &
		abs(E4 - avg.top) < 0.75/100 * avg.top &

		# bottoms E1,E3,E5 are within 0.75 percent of their average
		abs(E1 - avg.bop) < 0.75/100 * avg.bop &
		abs(E3 - avg.bop) < 0.75/100 * avg.bop &
		abs(E5 - avg.bop) < 0.75/100 * avg.bop &

		# lowest top > highest bottom
		min(E2,E4) > max(E1,E3,E5)
		})
	pattern$plot = expression({
		avg.top = (E2+E4)/2
		avg.bop = (E1+E3+E5)/3

		lines(c(d1,d3,d5), rep(avg.bop,3), lwd=10, col=col)
		lines(c(d2,d4), rep(avg.top,2), lwd=10, col=col)
			text(d3, max(E2,E4), 'RBOT', adj=c(0.5,0), xpd=TRUE)
		})
	patterns$RBOT = pattern

	#*****************************************************************
	# Analyzing Chart Patterns: Double Top And Double Bottom
	# http://www.investopedia.com/university/charts/charts4.asp
	#*****************************************************************
	# Double tops (DTOP), note in E and t first one is excluded
	#******************************************************************
	pattern = list()
	pattern$len = 3
	pattern$start = 'max'
	pattern$formula = expression({
		# Ea = max(E), ta = t[which.max(E)]
		second.top = max(E)
		second.top.t = t[which.max(E)]
		avg = (E1 + second.top)/2

		# E1 and Ea are within 1.5 percent of their average
		abs(E1         - avg) < 1.5/100 * avg &
		abs(second.top - avg) < 1.5/100 * avg &

		# ta - t1 > 22
		second.top.t - t1 > 22
		})
	pattern$plot = expression({
		second.top = max(E)
		second.top.d = d[which.max(E)]
		avg = (E1 + second.top)/2

		points(c(d1, second.top.d), c(E1, second.top), pch=2, lwd=2)
		lines(c(d1, second.top.d), rep(avg, 2), lwd=10, col=col)
			text(d2, avg, 'DTOP', adj=c(0.5,-0.5), xpd=TRUE)
		})
	pattern$last.point = expression(t[which.max(E)])
	patterns$DTOP = pattern

	#*****************************************************************
	# Double bottoms (DBOT)
	#******************************************************************
	pattern = list()
	pattern$len = 3
	pattern$start = 'min'
	pattern$formula = expression(
		# E1 and Ea = min(E) are within 1.5 percent of their average
		abs(E1 -         (E1+min(E))/2) < 1.5/100 * (E1+min(E))/2 &
		abs(max(E[-1]) - (E1+min(E))/2) < 1.5/100 * (E1+min(E))/2 &

		# ta - t1 > 22, ta = t[which.min(E)]
		t[which.min(E)] - t1 > 22
		)
	pattern$plot = expression({
		second.bot = min(E)
		second.bot.d = d[which.min(E)]
		avg = (E1 + second.bot)/2

		points(c(d1, second.bot.d), c(E1, second.bot), pch=2, lwd=2)
		lines(c(d1, second.bot.d), rep(avg, 2), lwd=10, col=col)
			text(d2, avg, 'DBOT', adj=c(0.5,1), xpd=TRUE)
		})
	pattern$last.point = expression(t[which.min(E)])
	patterns$DBOT = pattern

	# add name and convert start to +1/-1
	for(i in 1:len(patterns)) {
		patterns[[i]]$name = names(patterns)[i]
		patterns[[i]]$start = iif(patterns[[i]]$start == 'max', 1, -1)
	}

	return(patterns)
}


REBOL [
    Title: "Functions for converting and exchanging decimal values"
    File:  %decimal.r
    Author: "Eric Long"
    Email: kgd03011@nifty.ne.jp
    Co-Authors: ["Larry Palmiter" "Gerald Goertzel" "Oldes"]
    Date: 7-Jan-2012
    Category: [math util 4]
    Version: 1.0.1
    History: [
        1.0.0 15-Feb-2000 "Original Eric's version"
        1.0.1 7-Jan-2012 "Added real/to-native32 and real/from-native32 functions"
    ]
    Purpose: {
        Contains functions for the manipulation of decimal values,
        packaged into the REAL object. These provide full support for
        native binary floating-point file IO, compatible with C,
        FORTRAN, etc. REBOL decimal and money values may also be saved
        and loaded with no roundoff error. This package is necessary
        because the standard REBOL representation of decimal values
        may round off one or two significant digits, and the rounding
        of money values is an even greater source of error.

        This script contains two more objects, IEEE and EMPIRICAL,
        which provide several functions useful for exploring how
        decimal values are represented as double floating point numbers,
        and for testing the inexactness of the REBOL comparison functions.
    }
    Dependencies: [
        {The following require %format.r to properly format the output,
        though they will work without it. %format.r should be loaded
        before this script.}

        real/form
        real/show/decimal
        IEEE/rebtest
        empirical/rebtest

        {please download: } http://www.rebol.org/utility/format.r
    ]
]

comment { =============== REAL ===============

REAL is an object that packages several functions useful for examining,
saving, exporting and importing decimal values:

Main Interface Functions
FORM           Returns precise decimal representation (requires format.r).
SHOW           Returns a string showing the bits in the IEEE representation.
TO-NATIVE      Converts a decimal value into a native IEEE binary
FROM-NATIVE    Converts an 8-byte native binary into a decimal value.
SAVE           Saves a value with all decimal values converted.
LOAD           Loads a value and restores 8-byte binaries to decimals.
WRITE          Writes a block of decimal values to a file in native form.
READ           Reads a file containing a series of decimal values expressed
               in native form.

Helper Functions
SPLIT          Returns a block with the three components of the IEEE
               double floating point representation of a decimal value.
CONVERT        Converts all decimal values to native binaries, leaves other
               values untouched.
RESTORE        Restores 8-byte binaries to decimal values.
TO-BIN         Returns a binary string representation or a numeric value.
FROM-BIN       Returns the integer represented by string of 1's and 0's.
TO-MATRIX      Converts a flat block into nested blocks of any depth.

}

real: make object! [

comment { =============== REAL/FORM ===============

NOTE: Please see FULL-FORM in format.r for details.

}

; Set the word FORM in the context of this object:
; either to the function FULL-FORM if available, or else to the global FORM.

form: either value? 'full-form [:full-form][get in system/words 'form]

comment { =============== REAL/SHOW ===============

EXAMPLES:

>> real/show 2
== {0 10000000000 0000000000000000000000000000000000000000000000000000}
>> real/show 3
== {0 10000000000 1000000000000000000000000000000000000000000000000000}
>> real/show 3.5
== {0 10000000000 1100000000000000000000000000000000000000000000000000}
>> real/show 4
== {0 10000000001 0000000000000000000000000000000000000000000000000000}
>> real/show/decimal 3.5
== "0 1024 3377699720527872"          ; requires format.r

>> real/show ieee/max-real
== {0 11111111110 1111111111111111111111111111111111111111111111111111}

}

show: func [
    "return an IEEE-compatible string representation in binary"
    x  [number!]
    /decimal  "return a decimal representation"
    /local out sign exponent fraction
][
    set [sign exponent fraction] split x
    out: copy "  "
    either decimal [
        append out form fraction
        insert next out form exponent
        insert out sign
    ][
        append out to-bin/length fraction 52
        insert next out to-bin/length exponent 11
        insert out sign
    ]
    out
]

comment { =============== REAL/TO-NATIVE REAL/FROM-NATIVE ===============

EXAMPLES:

>> real/to-native pi
== #{400921FB54442D18}
>> real/to-native/rev pi
== #{182D4454FB210940}            ; configuration used in PC's
>> real/from-native real/to-native pi
== 3.14159265358979
>> pi - real/from-native real/to-native pi
== 0                              ; converts back to exact value
}

to-native: func [
    "convert a numerical value into native binary format"
    x  [number!]
    /rev     "reverse binary output"
    /local out sign exponent fraction
][
    set [sign exponent fraction] split x
    out: copy #{}
    loop 6 [
        insert out to char! byte: fraction // 256
        fraction: fraction - byte / 256
    ]
    insert out to char! exponent // 16 * 16  + fraction
    insert out to char! exponent / 16 + (128 * sign)
    return either rev [head reverse out][out]
]
to-native32: func [
    "convert a numerical value into native binary format"
    x  [number!]
    /rev     "reverse binary output"
    /local out sign exponent fraction
][
    set [sign exponent fraction] split32 x
    out: copy #{}
    loop 2 [
        insert out to char! byte: fraction // 256
        fraction: fraction - byte / 256
    ]
    insert out to char! exponent * 128 // 256  + fraction
    insert out to char! exponent / 2 + (128 * sign)
    return either rev [head reverse out][out]
]
from-native: func [
    {convert a binary native into a decimal value - also accepts a binary
    string representation in the format returned by REAL/SHOW}
    in      [binary! string!]
    /rev    "binary input in reverse order"
    /local sign exponent fraction
][
    in: copy in
    either binary? in [
        if rev [reverse in]
        sign: either zero? to integer! (first in) / 128 [1][-1]
        exponent: (first in) // 128 * 16 + to integer! (second in) / 16
        fraction: to decimal! (second in) // 16
        in: skip in 2
        loop 6 [
            fraction: fraction * 256 + first in
            in: next in
        ]
    ][
        set [sign exponent fraction] parse in none
        sign: either sign = "0" [1][-1]
        exponent: from-bin exponent
        fraction: from-bin fraction
    ]
    sign * either zero? exponent [
        2 ** -1074 * fraction
    ][
        2 ** (exponent - 1023) * (2 ** -52 * fraction + 1)
    ]
]
from-native32: func [
    {convert a binary native into a decimal value - also accepts a binary
    string representation in the format returned by REAL/SHOW}
    in      [binary! string!]
    /rev    "binary input in reverse order"
    /local sign exponent fraction
][
    in: copy in
    either binary? in [
        if rev [reverse in]
        sign: either zero? to integer! (first in) / 128 [1][-1]
        exponent: ((first in) and 127) * 2 + to integer! (second in) / 128
        fraction: to decimal! (second in) // 128
        in: skip in 2
        loop 2 [
            fraction: fraction * 256 + first in
            in: next in
        ]
    ][
        set [sign exponent fraction] parse in none
        sign: either sign = "0" [1][-1]
        exponent: from-bin exponent
        fraction: from-bin fraction
    ]
    sign * either zero? exponent [
        2 ** -149 * fraction
    ][
        2 ** (exponent - 127) * (2 ** -23 * fraction + 1)
    ]
]
comment { =============== REAL/SAVE REAL/LOAD ===============

REAL/SAVE and REAL/LOAD are meant to be completely compatible
with the standard functions SAVE and LOAD, except that there
will be no roundoff error from saving and loading decimal
or money values.

EXAMPLE:

>> amount: [100 * $1.504]
== [100 * $1.50]

>> save %test-x.dat amount
>> do load %test-x.dat
== $150.00                             ; large round-off error

>> real/save %test-y.dat amount
>> do real/load %test-y.dat
== $150.40                             ; no error
}

save: func [
    {Saves a value with all decimals converted to binary natives}
    where [file! url!] "Where to save it."
    value              "Value to save."
    /header            "Save it with a header"
        header-data [block! object!] "Header block or object"
][
    either header [
        system/words/save/header where convert :value convert header-data
    ][
        system/words/save where convert :value
    ]
]

load: func [
    {Loads a value and converts binary natives to decimals}
    source [file! url! string! any-block!]
    /header "Includes REBOL header object if present"
    /next   {Load the next value only.
            Return block with value and new position.}
    /local lp
][
    lp: copy 'system/words/load
    foreach item [header next][
        if get item [insert tail :lp item]
    ]
    restore lp source
]

comment { =============== REAL/WRITE REAL/READ ===============

NOTE: REAL/WRITE and REAL/READ are meant for writing and reading
      decimal values in native binary format. REAL/WRITE accepts
      a single numerical value or a block of numerical values that
      may be nested to any depth, but the return value of REAL/READ
      is always a flat block.

EXAMPLES:

>> b1: [] for x 1 16 1 [append b1 square-root x]
== [1 1.4142135623731 1.73205080756888 2 2.23606797749979 ...
>> real/write %test.bin b1
16 decimal values written

>> bb1: real/read %test.bin
== [1 1.4142135623731 1.73205080756888 2 2.23606797749979 ...
>> b1/7 - probe bb1/7
2.64575131106459                ; square root of 7
== 0                            ; is restored exactly

>> b2: real/to-matrix b1 4      ; convert to 4x4 matrix
== [[1 1.4142135623731 1.73205080756888 2] [2.23606797749979 ...
>> real/write %test.bin b2
16 decimal values written       ; the same values are written,
>> bb2: real/read %test.bin     ; and read back into a flat block
== [1 1.4142135623731 1.73205080756888 2 2.23606797749979 ...
}

write: func [
    {Writes a (nested) block of decimal values
    to a file in native binary format}
    f [file!]          "file to write to"
    b [block! number!] "number or block of numbers"
    /rev               "reverse bytes if needed"
    /append            "append to file"
    /local out elem p to-do
][
    out: copy #{}
    to-do: copy []
    insert to-do b
    nat: to path! either rev [[to-native rev]][[to-native]]
    while [ not tail? to-do ] [
        either block? first to-do [
            change/part to-do first to-do 1
        ][
            insert tail out nat first to-do
            remove to-do
        ]
    ]
    either append [
        system/words/write/binary/append f out
    ][
        system/words/write/binary f out
    ]
    print [(length? out) / 8  "decimal values written"]
]

read: func [
    {Returns a block of numbers read in from a binary data file}
    f "file name of binary file"
    /rev "reverse bytes if needed"
    /local tmp out from l
][
    tmp: system/words/read/binary f
    l: (length? tmp) / 8
    if not zero? l - to integer! l
        [make error! "read-native: bad file size"]
    out: make block! l
    from: make path! either rev [[from-native rev]][[from-native]]
    loop l [
        insert tail out from copy/part tmp 8
        tmp: skip tmp 8
    ]
    out
]

comment { =============== REAL/SPLIT ===============

SPLIT calculates the three components of the IEEE representation of a
double floating point value. These are the actual values used by the CPU
in numerical calculations.

>> set [sign exponent fraction] real/split pi
== [0 1024 2.57063812465794E+15]
>> (-1 ** sign) * (2 ** (exponent - 1023)) * (1 + (fraction * (2 ** -52)))
== 3.14159265358979

Numbers smaller than 2 ** -1022 (denormals, which have an exponent
component of zero) use a different formula:

>> set [sign exponent fraction] real/split probe 2 ** -1030
8.69169475979376E-311
== [0 0 17592186044416]
>> (-1 ** sign) * (2 ** -1074) * fraction
== 8.69169475979376E-311
}

split: func [
    "Returns block containing three components of double floating point value"
    x [number!] /local sign exponent fraction
][
    sign: either negative? x [x: (- x) 1][0]

    either zero? x [exponent: 0  fraction: 0][

        either zero? 1024 - exponent: to integer! log-2 x [exponent: 1023][
            if positive? (2 ** exponent) - x [exponent: exponent - 1]
        ]
        fraction: x / (2 ** exponent)

        either positive? exponent: exponent + 1023 [
            fraction: fraction - 1         ; drop the first bit for normals
            fraction: fraction * (2 ** 52) ; make the remaining fraction an
                                           ; "integer"
        ][
            fraction: 2 ** (51 + exponent) * fraction  ; denormals
            exponent: 0
        ]
    ]
    reduce [sign exponent fraction]
]

split32: func [
    "Returns block containing three components of double floating point value"
    x [number!] /local sign exponent fraction
][
    sign: either negative? x [x: (- x) 1][0]

    either zero? x [exponent: 0  fraction: 0][

        either zero? 128 - exponent: to integer! log-2 x [exponent: 127][
            if positive? (2 ** exponent) - x [exponent: exponent - 1]
        ]
        fraction: x / (2 ** exponent)

        either positive? exponent: exponent + 127 [
            fraction: fraction - 1         ; drop the first bit for normals
            fraction: fraction * (2 ** 23) ; make the remaining fraction an
                                           ; "integer"
        ][
            fraction: 2 ** (22 + exponent) * fraction  ; denormals
            exponent: 0
        ]
        fraction: to integer! fraction + .5
    ]
    reduce [sign exponent fraction]
]
comment { =============== REAL/CONVERT REAL/RESTORE ===============

These functions are principally meant to be used by REAL/SAVE and
REAL/LOAD. REAL/CONVERT accepts a value of any SAVE-able datatype,
converts all decimal values to native binaries, and returns the
result. The argument value is unaffected. REAL/RESTORE does the
reverse of REAL/CONVERT, restoring all of the original decimal
values. This allows decimal values to be saved with no roundoff error.

Since all object and any-block values are copied, the result of
converting and restoring may not be identical to the original value.
The result will be good enough, however, to allow saving and
loading of values compatible with the behavior of the standard
functions SAVE and LOAD, and without any roundoff error.

EXAMPLE:

>> circle-area: func [r [number!]] compose [(pi) * r * r]
>> probe obj1: make object! compose/deep [
[    a: [(pi) (exp 1)] b: (EU$1.00 * pi) c: real/to-native (pi)
[    f: :circle-area ]

make object! [
    a: [3.14159265358979 2.71828182845905]
    b: EU$3.14
    c: #{400921FB54442D18}
    f: func [r [number!]][3.14159265358979 * r * r]
]

>> probe obj2: real/convert obj1

make object! [
a: [#{400921FB54442D18} #{4005BF0A8B145769}]
b: (EU$1.00 * #{400921FB54442D18})   ; money value converted to paren
c: ['escape #{400921FB54442D18}]     ; pre-existing 8-byte binary is escaped
f: func [r [number!]][#{400921FB54442D18} * r * r]
]
>> probe obj3: real/restore obj2

make object! [
a: [3.14159265358979 2.71828182845905]
b: EU$3.14
c: #{400921FB54442D18}
f: func [r [number!]][3.14159265358979 * r * r]
]

>> obj3/a/1 - pi                   ; decimal values are exactly restored
== 0
>> (obj3/f 4) - circle-area 4      ; restored function gives same answer
== 0

}

convert: func [
    {returns (copy of) B with all decimals converted to binary natives}
    value     "value to convert"
    /local r
][
    if function? :value [
        return func load mold third :value convert second :value
    ]
    if object? :value [
        r: make value []
        foreach word next first r [
            set in r word convert get in r word
        ]
        return r
    ]
    if any-block? :value [
        r: copy :value
        while [not tail? :r] [
            change/only :r convert first :r
            r: next :r
        ]
        return head :r
    ]
    if money? :value [
        return to paren! compose [(value / value/2) * (to-native value/2)]
    ]
    if all [
        binary? :value
        8 = length? value
    ][                            ; return an "escaped" form
        return compose ['escape (value)]
    ]
    if :value = first ['escape] [
        return ['escape 'escape]
    ]
    either decimal? :value [to-native value][:value]
]

restore: func [
    {returns (copy of) B with all binary natives converted to decimals}
    value     "value to convert"
    /local r item
][
    if function? :value [
        return func load mold third :value restore second :value
    ]
    if object? :value [
        r: make value []
        foreach word next first r [  ; cycle through all words except first
            set in r word restore get in r word
        ]
        return r
    ]
    if any-block? :value [
        if all [
            paren? :value
            3 = length? :value
            money? first :value
            binary? item: pick :value 3
        ][                        ; restore a money value
            change at :value 3 from-native item
            return value
        ]
        if all [
            block? :value
            2 = length? value
            (first value) = (first ['escape])
        ][
            return second value   ; return the "escaped" value
        ]
        r: copy :value
        while [not tail? :r] [
            change/only :r restore first :r
            r: next :r
        ]
        return head :r
    ]
    either all [
        binary? :value
        8 = length? value
    ][from-native value][:value]
]

comment { =============== TO-BIN FROM-BIN ===============

NOTE: used by REAL/SHOW and REAL/FROM-NATIVE

}

to-bin: func [
    {return a binary string representation of N}
    n [number!] "number to show in binary (integer value from 0 to 2 ** 53)"
    /length l   "length of string desired"
    /local r v
][
    if not l [l: 1]
    either negative? n - 1 [r: copy "0"][
        r: copy ""
        v: 2 ** to integer! log-2 n
        if negative? n - v [v: v / 2]  ; in case LOG-2 rounds up
        while [ v >= 1 ] [
            insert tail r either negative? n - v ["0"][n: n - v "1"]
            v: v / 2
        ]
    ]
    head insert/dup r "0" (l - length? r)
]

from-bin: func [
    {convert a binary string representation into a numeric value}
    b [string!]
    /local x  bitval
][
    bitval: 1.0
    b: tail b
    x: 0
    while [ not head? b ] [
        b: back b
        if #"1" = first b [x: x + bitval]
        bitval: bitval * 2
    ]
    x
]

comment { =============== TO-MATRIX ===============

EXAMPLE:

>> to-matrix [1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16] 4
== [[1 2 3 4] [5 6 7 8] [9 10 11 12] [13 14 15 16]]
>> to-matrix [1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16] [2 4]
== [[[1 2 3 4] [5 6 7 8]] [[9 10 11 12] [13 14 15 16]]]

}

to-matrix: func [
    {Returns a matrix constructed from the values of B}
    b [block!]
    row-length [number! block!]
    /local out number-rows
][
    if number? row-length [row-length: reduce [row-length]]
    reverse row-length
    number-rows: length? b
    foreach l row-length [
        number-rows: number-rows / l
        if not zero? number-rows - to integer! number-rows
            [make error! "to-matrix: bad dimensions"]
        out: make block! number-rows
        loop number-rows [
            insert/only tail out copy/part b l
            b: skip b l
        ]
        b: out
    ]
    out
]

]  ; end REAL

comment { =============== IEEE ===============

IEEE is an object that contains:

REBTEST   A function that tests the sensitivity of the REBOL EQUAL?
          comparison function.
GET-NEXT  A function that makes the minimum increment to a decimal value.
GET-LAST  A function that makes the maximum decrement to a decimal value.
MAX-REAL  The largest decimal value.
MIN-NORM  The smallest positive normalized decimal.
MIN-REAL  The smallest positive decimal value.
EPS       The minimum increment to 1 (smallest x such that (x+1)>1).

All of the increments and other values in IEEE are calculated based on
IEEE Std 754 (International Electrical and Electronic Engineering society).

REBTEST points out the inexactness of the REBOL comparison functions (as of
version 2.2), but note that it is easy to get exact comparisons by using
ZERO? NEGATIVE? and POSITIVE? to test the results of subtraction:

>> x: .1
== 0.1
>> y: ieee/get-next .1     ; make minimum increment to .1
== 0.1
>> y - x
== 1.38777878078145E-17    ; Y is greater than X ...
>> equal? x y
== true                    ; inexact
>> zero? x - y
== false                   ; exact
>> greater? y x
== false
>> positive? y - x
== true
>> lesser? x y
== false
>> negative? x - y
== true

; Example of rebtest (format.r not loaded so we get REBOL display values)
; Let's use 2 ** 53 as the target. All positive integers from 1 to 2 ** 53
; are exact in binary form. All doubles greater than 2 ** 52 are integers.
; The doubles from 2 ** 52 thru 2 ** 53 are consecutive integers.
; The first column shows the result of REBOL's test for equality of the target
; to (target + increment). The 2nd column shows REBOL display value and the
; 3rd column shows the increment.

>> ieee/rebtest 2 ** 53             ; the exact value is 9007199254740992
target 9.00719925474099E+15         ; REBOL display rounds to 15 sig. digits
positive increments
t=t+i target+increment increment
true 9.00719925474099E+15 2         ; positive increments are of size 2
true 9.007199254741E+15 4         ; REBOL display value changes
true 9.007199254741E+15 6         ; REBOL equality test is wrong
true 9.007199254741E+15 8
true 9.007199254741E+15 10
true 9.007199254741E+15 12
true 9.00719925474101E+15 14     ; REBOL display changes again
true 9.00719925474101E+15 16     ; but equality test is still wrong
false 9.00719925474101E+15 18     ; equality test correct once we add 18
false 9.00719925474101E+15 20
negative increments
t=t+i target+increment increment
false 9.00719925474099E+15 -1    ; negative increments are of size 1
false 9.00719925474099E+15 -2
false 9.00719925474099E+15 -3
false 9.00719925474099E+15 -4
false 9.00719925474099E+15 -5
false 9.00719925474099E+15 -6     ; REBOL equality test OK for all inc.
false 9.00719925474099E+15 -7
false 9.00719925474098E+15 -8     ; REBOL display changes
false 9.00719925474098E+15 -9
false 9.00719925474098E+15 -10

}

IEEE: make object! [

comment { =============== IEEE/REBTEST ===============

REBTEST makes ten successive minimum increments and decrements to the
target value, and tests whether EQUAL? returns TRUE when comparing
the results to the original value. In all cases EQUAL? should return
false.

}

rebtest: func [
     {displays 21 adjacent doubles centered on x}
     x [number!] "target value"
     /local y
][
    print ["target"  x ]
    print "positive increments"
    print ["t=t+i"  "target+increment"  "increment"]
    y: x
    loop 10 [
        if all [positive? y  zero? y - max-real][
            print ["" "+Inf"]
            break
        ]
        y: get-next y
        print [equal? x y  y  y - x]
    ]
    print "negative increments"
    print ["t=t+i"  "target+increment"  "increment"]
    y: x
    loop 10 [
        if all [negative? y  zero? y + max-real][
            print ["" "-Inf"]
            break
        ]
        y: get-last y
        print [equal? x y  y  y - x]
    ]
]

get-next: func [
    {returns next double after x}
    x [number!] "input value"
    /local exp inc
][
    if zero? x [return 2 ** -1074]
    if negative? x [return - get-last (- x)]
    either zero? 1024 - exp: to-integer log-2 x [exp: 1023][
        if positive? (2 ** exp) - x [exp: exp - 1]
    ]
    inc: 2 ** (exp - 52)                ; local machine precision
    if exp < -1021 [inc: 2 ** -1074]    ; handle denormals
    x + inc
]

get-last: func [
    {returns next double before x}
    x [number!] "input value"
    /local exp inc
][
    if zero? x [return - (2 ** -1074)]
    if negative? x [return - get-next (- x)]
    either zero? 1024 - exp: to-integer log-2 x [exp: 1023][
        if not negative? (2 ** exp) - x [exp: exp - 1]
    ]
    inc: 2 ** (exp - 52)                ; local machine precision
    if exp < -1021 [inc: 2 ** -1074]    ; handle denormals
    x - inc
]

max-real: 2 ** 1023 * (2 - (2 ** -52))   ; maximum decimal value

min-norm: 2 ** -1022         ; minimum normalized positive value

min-real: 2 ** -1074         ; minimum positive decimal value

eps: 2 ** -52   ; machine epsilon (smallest x such that (x+1)>1)

; if possible, set PRINT locally to a function providing formatted output

print: either value? 'format [
    func [line][
        system/words/print format/full reduce line [#8..3 #24.16.1 #12.2.1]
    ]
][
    get in system/words 'print
]

] ; end IEEE

comment { =============== EMPIRICAL ===============

EMPIRICAL is equivalent to IEEE, but whereas the values in IEEE are
calculated based on the IEEE standard, the values in EMPIRICAL are
obtained through an empirical algorithm. The function CMP-SENS does
this by actually trying a series of increments to the target value,
until the smallest value is identified that results in a different
value when added to the target.

}

empirical: make object! [

cmp-sens: func [
    {find smallest effective increment for TARGET.
    NOTE: this is less than the resulting increment}
    target [number!]
    /exact   {results for exact comparison - default is use EQUAL?}
    /below   "return decrement (default is increment)"
    /local diff upper lower last-diff eqf operation no-effect
][
    eqf: either exact [func[x y][zero? x - y]][:equal?]
    operation: either below [ :subtract ][ :add ]
    last-diff: upper: max   2 ** -30   abs target * (2 ** -30)
    lower: 0
    diff:  upper + lower / 2
    while [ not zero? last-diff - diff ] [
        last-diff: diff
        if error? try [
            no-effect: eqf target operation target diff
        ][
            no-effect: none
        ]
        either no-effect [
            lower: diff
            diff: diff + upper / 2
        ][
            upper: diff
            diff: diff + lower / 2
        ]
    ]
    return either none? no-effect [none][upper]
]

rebtest: func [
    {compare sensitivity of REBOL comparison operators against
    actual increment value}
    target [number!]
    /local inc rebsense factor
][
    if inc: cmp-sens/exact target [             ; get effective increment
        inc: target + inc - target              ; get resulting increment
        if rebsense: cmp-sens target [
            rebsense: target + rebsense - target
            if 1e10 < factor: rebsense / inc [factor: form factor]
        ]
    ]
    print ["Increments:"  "Smallest" "NOT EQUAL?" "Factor"]
    print ["Positive:" inc  rebsense factor]
    factor: none
    if inc: cmp-sens/below/exact target [
        inc: target - inc - target
        if rebsense: cmp-sens/below target [
            rebsense: target - rebsense - target
            if 1e10 < factor: rebsense / inc [factor: form factor]
        ]
    ]
    print ["Negative:" inc rebsense factor]
]

get-next: func [
    {make the smallest possible increment to a double}
    target [number!]
    /count n "number of times to do this"
][
    if not n [n: 1]
    loop n [
        target: target + cmp-sens/exact target
    ]
]

get-last: func [
    {make the smallest possible decrement to a double}
    target [number!]
    /count n "number of times to do this"
][
    if not n [n: 1]
    loop n [
        target: target - cmp-sens/below/exact target
    ]
]

max-real: func [
    {returns the highest numerical value on the system}
    /local x lower upper factor last-factor
][
    x: 2.0
    loop 100000 [
        if error? try [x: x * 2][break]
    ]
    lower: 1
    upper: 2
    last-factor: 0
    loop 100000 [
        if zero? last-factor - factor: lower + upper / 2 [break]
        either error? try [x * factor] [upper: factor][lower: factor]
        last-factor: factor
    ]
    x * lower
]

min-real: get-next 0

eps: (get-next 1) - 1

print: either value? 'format [
    func [
        {print out a formatted line if FORMAT is available}
        line
    ][
        system/words/print format/full reduce line
            [#-12 #12.2.1 #12.2.1 #8]
    ]
][
    get in system/words 'print
]

]  ; end empirical


# Function for calculate phytosociological table
# Version: 1.2.0
# https://github.com/MarioJose/r-functions/tree/master/phyto

phyto <- function(x, filter = NULL, area = NULL, criteria = NULL, measure = NULL, incDead = TRUE, nmDead = "Dead", diversity = TRUE, evenness = TRUE){
  # x must be data frame with: plot, family, specie, diameter, (height).
  #   It must be in this order, but not necessarily with this names. Height is
  #   optional

  # Checking input
  # ++++++++++++++

  if(!is.data.frame(x)){
    stop("'x' must be a data frame")
  } else {
    if(dim(x)[2] < 4 | dim(x)[2] > 5){
      stop("Your data frame must have at least 4 columns with the follow data at the same order: 'plot', 'family', 'specie', 'diameter'. The column 'height' is optional")
    }
  }

  if(is.null(filter)){
    stop("You must inform the filter to summarization: 'plot', 'family', 'genus', 'specie'")
  } else {
    if(!(filter %in% c("plot", "family", "genus", "specie"))){
      stop("You must inform one of the follow option to filter: 'plot', 'family', 'genus', 'specie'")
    }
  }

  if(is.null(area)){
    stop("You must inform the area sampled")
  }

  if(is.null(criteria)){
    stop("You must inform the diameter criteria for individual inclusion")
  }

  if(is.null(measure)){
    stop("You must inform the measure: \"d\" = diameter; \"c\" = circumference")
  } else {
    if(!(measure %in% c("d","c"))){
      stop("You must inform one of the follow option to measure: \"d\" = diameter; \"c\"c = circumference")
    }
  }

  # Convert family and species column to character
  x[ ,2] <- as.character(x[ ,2])
  x[ ,3] <- as.character(x[ ,3])

  # Remove dead individuals (column 'specie')
  if(!incDead){
    didx <- grep(nmDead , x[ ,3])
    x <- x[-didx, ]
    print(paste("Removed", length(didx), "individual named as", nmDead))
  }

  # Functions
  # +++++++++

  # Split multiple diameter or height and return diameter of total basal area of
  # each individual or mean height of each individual
  splitMultiple <- function(x, m){

    if(!is.numeric(x)) tmp <- as.numeric(unlist(strsplit(x , "+", TRUE)))
    else tmp <- x

    # Expression used in conversion
    # area = (pi*diameter^2)/4
    # area = (circumference^2)/(4*pi)
    # diameter = sqrt((4*area)/pi)

    # Return diameter of total basal area
    if(m == "d") out <- sqrt(4 * sum((pi * (tmp ^ 2)) / 4) / pi)

    # Convert circumference and return diameter of total basal area
    if(m == "c") out <- sqrt(4 * sum((tmp ^ 2) / (4 * pi)) / pi)

    # Return mean of height
    if(m == "h") out <- mean(tmp)

    return(out)
  }

  # Shannon and Simpson diversity index
  shannon_fn <- function(x){
    p <- x / sum(x)
    H <- -sum(p * log(p))
    # Variance as Hutcheson (1970)
    varH <- ((sum(p * (log(p)^2)) - sum(p * log(p))^2) / sum(x)) + ((length(x) - 1) / (2 * (length(x)^2)))

    return(c(H = H, varH = varH))
  }

  simpson_fn <- function(x, var = FALSE){
    # As Simpson (1949)
    N <- sum(x)
    p <- x / N
    D <- sum(x * (x - 1)) / (N * (N - 1))
    varD <- ( 4*N*(N - 1)*(N - 2)*sum(p^3) + 2*N*(N - 1)*sum(p^2) - 2*N*(N - 1)*(2*N - 3)*(sum(p^2)^2) ) / (N*(N - 1))^2
    # if N be very large, approximately
    #varD <- (4/N) * ( (sum(p^3)) - (sum(p^2))^2 );

    return(c(D = D, varD = varD))
  }

  # Evar and E1/D evenness index
  evenness_fn <- function(x){
    # As Smith & Wilson (1996)

    mulog <- sum(log(x)) / length(x)
    # 0 = minimum eveness
    Evar <- 1 - (2 / pi) * atan( sum((log(x) - mulog)^2) / length(x) )
    E1D <- (1 / simpson_fn(x)[["D"]]) / length(x)

    return(c(Evar = Evar, E1D = E1D))
  }

  # Checking data
  # +++++++++++++

  # Columns names
  colnames(x) <- c("plot", "family", "specie", "diameter", "height")[1:dim(x)[2]]

  # Create column with genus
  x$genus <- sapply(x$specie, function(x) strsplit(x, " ")[[1]][1])

  # Order data frame
  x <- x[, c(1:2, dim(x)[2], 3:(dim(x)[2] - 1))]

  # Standardize measure to diameter. Split multiples measures and return
  # diameter of total basal area to individual
  if(is.character(x$diameter) | is.factor(x$diameter))
    x$diameter <- sapply(as.character(x$diameter), splitMultiple, m = measure)
  else if(measure == "c") x$diameter <- x$diameter / pi

  # Standardize height. Split multiples measures and return mean of height to
  #   individual
  if(dim(x)[2] == 6)
    if(is.character(x$height) | is.factor(x$height))
      x$height <- sapply(as.character(x$height), splitMultiple, m = "h")

  # Filter by inclusion criteria
  cidx <- x$diameter >= criteria
  x <- x[cidx, ]

  if(dim(x)[1] < 1) stop("Your criteria removed all individuals")
  else print(paste("Removed", length(cidx) - sum(cidx), "individual(s) by criteria", criteria))

  # Calculate parameters
  # ++++++++++++++++++++

  # Create output data frame
  out <- aggregate(list(nInd = x$specie), by = list(c1 = x[ ,filter]), FUN = length)
  colnames(out) <- c(filter, "nInd")

  # Add families
  if(filter %in% c("genus", "specie")){
    tf <- aggregate(list(nInd = x[ ,filter]), by = list(family = x$family, filter = x[ ,filter]), FUN = length)
    out$family <- tf$family[match(out[ ,filter], tf$filter)]
    out <- out[ ,c("family", filter, "nInd")]
  }

  if(filter == "plot"){
    # Number of families per plot
    out[ ,"nFamilies"] <- aggregate(x$family, by = list(x$plot), FUN = function(x){length(unique(x))})$x

    # Number of genera per plot
    out[ ,"nGenera"] <- aggregate(x$genus, by = list(x$plot), FUN = function(x){length(unique(x))})$x

    # Number of species per plot
    out[ ,"nSpecies"] <- aggregate(x$specie, by = list(x$plot), FUN = function(x){length(unique(x))})$x

    if(diversity){
      out[ ,c("H", "varH")] <- aggregate(x$specie, by = list(x$plot), FUN = function(x){shannon_fn(table(as.character(x)))})[[2]]
      out[ ,c("D", "varD")] <- aggregate(x$specie, by = list(x$plot), FUN = function(x){simpson_fn(table(as.character(x)))})[[2]]
    }

    if(evenness)
      out[ ,c("Evar","E1D")] <- aggregate(x$specie, by = list(x$plot), FUN = function(x){evenness_fn(table(as.character(x)))})[[2]]

  }

  if(filter == "family"){
    # Number of genera per family
    out[ ,"nGenera"] <- aggregate(x$genus, by = list(x$family), FUN = function(x){length(unique(x))})$x

    # Number of species per family
    out[ ,"nSpecies"] <- aggregate(x$specie, by = list(x$family), FUN = function(x){length(unique(x))})$x
  }

  if(filter == "genus"){
    # Number of species per genus
    out[ ,"nSpecies"] <- aggregate(x$specie, by = list(x$genus), FUN = function(x){length(unique(x))})$x
  }

  # Convert diameter measure from centimetres to meters
  x$diameter <- x$diameter / 100
  out[ ,"tBasalArea"] <- aggregate(x$diameter, by = list(x[ ,filter]), FUN = function(x){sum((pi * (x^2)) / 4)})$x

  if(filter != "plot"){
    out[ ,"AbsDens"] <- out$nInd / area
    out[ ,"RelDens"] <- (out$nInd / sum(out$nInd)) * 100

    # Species or genera by plot
    tmp <- aggregate(list(nInd = x[ ,filter]), by = list(filter = x[ ,filter], plot=x$plot), FUN = length)
    out[ ,"nPlot"] <- aggregate(tmp$filter, by = list(tmp$filter), FUN = length)$x
    out[ ,"AbsFreq"] <- (out[ ,"nPlot"] / length(unique(x$plot))) * 100
    out[ ,"RelFreq"] <- (out$AbsFreq / sum(out$AbsFreq)) * 100

    out[ ,"AbsDom"] <-  out[ ,"tBasalArea"] / area
    out[ ,"RelDom"] <- (out[ ,"tBasalArea"] / sum(out[ ,"tBasalArea"])) * 100

    out[ ,"IVI"] <- out$RelDens + out$RelFreq + out$RelDom
    out[ ,"CVI"] <- out$RelDens + out$RelDom
  }

  if(length(x[1, ]) == 6){
    out[ ,"meanHeight"] <- aggregate(x[ ,6], by = list(x[ ,filter]), FUN = mean)$x
    out[ ,"sdHeight"] <- aggregate(x[ ,6], by = list(x[ ,filter]), FUN = sd)$x
  }

  return(out)
}


# Internal function that uses the implementation of Telfer written by Gary Powney

telfer_func <- function (taxa_data, iterations = 10, useIterations = TRUE, minSite = 5){

  # Check we have two time periods
  TPs <- sort(unique(taxa_data$time_period))
  if(length(unique(taxa_data$time_period)) != 2) stop('There are not two time periods in the data')

  # subset data to one object for each time perod
  T1 <- unique(taxa_data[taxa_data$time_period == TPs[1], c('taxa', 'site')])
  T2 <- unique(taxa_data[taxa_data$time_period == TPs[2], c('taxa', 'site')])

  # remove gridcells that are only found in one time period
  T1cells <- unique(T1$site)
  T2cells <- unique(T2$site)
  allGood <- T1cells[T1cells %in% T2cells]
  T1 <- T1[T1$site %in% allGood,]
  T2 <- T2[T2$site %in% allGood,]

  ### Identify the number of sites each taxa occupies in each time period. ###
  T1_range <- data.frame(taxa = names(table(T1$taxa)), T1Nsites = as.numeric(table(T1$taxa)))
  T2_range <- data.frame(taxa = names(table(T2$taxa)), T2Nsites = as.numeric(table(T2$taxa)))

  ### Remove taxa which have less than minSite grid cells in first period ###
  T1_range_good <- T1_range[T1_range$T1Nsites >= minSite,]
  if(nrow(T1_range_good) == 0) stop(paste('No taxa satisfy the minSite criteria when comparing time',
                                          'period', TPs[1], 'and', TPs[2]))

  ### Link the two tables by taxa ###
  spp_table <- merge(T1_range_good, T2_range, by = 'taxa')

  ### Identify the simple difference between the two time periods ###
  spp_table$range_change <- spp_table$T2Nsites - spp_table$T1Nsites

  ### convert the grid cell number to proportion of total number of cells surveyed ###
  total.cells <- length(allGood)

  # To avoid the problems associated with 0 proportions they were calculated as (x + 0.5) / (n + 1).
  spp_table$T1_range_prop <- (spp_table$T1Nsites + 0.5) / (total.cells + 1)
  spp_table$T2_range_prop <- (spp_table$T2Nsites + 0.5) / (total.cells + 1)

  spp_table$T1_logit_range <- log(spp_table$T1_range_prop / (1 - spp_table$T1_range_prop))  # logit transform the proportions
  spp_table$T2_logit_range <- log(spp_table$T2_range_prop / (1 - spp_table$T2_range_prop))	# logit transform the proportions

  ### To account for non constant variance we must do the following steps to create a variable to weight the final regression ###
  row.names(spp_table) <- spp_table$taxa # name rows helps make sense of the model output
  m1 <- lm(T2_logit_range ~ T1_logit_range, data = spp_table)  	# linear regression of two time periods

  if(!useIterations){

    spp_table$change_index <- rstandard(m1)
    final_output_table <- spp_table[,c(1:3,9)]
    return(list(final_output_table, spp_table))

  } else if(useIterations){

    spp_table$sq_residual_m1 <- resid(m1)^2 # square the residuals
    spp_table$fitted_proportions <- ilt(m1$fitted) # exponential function, overwrite fitted values to fitted proportions

    total.cell.1 <- total.cells + 1								# total grid cells surveyed + 1

    spp_table$NP_test <- 1 / (total.cell.1 * spp_table$fitted_proportions *
                                (1 - spp_table$fitted_proportions))	# 1 / [NP (1 - P)] in telfer paper

    # second model which is the squared residuals of m1 on 1/[NP (1-P)]
    m2 <- lm(sq_residual_m1 ~ NP_test, data = spp_table)

    # second model which is the squared residuals of m1 on 1/[NP (1-P)]
    c_ <- coef(m2)[1] # take the intercept of m2
    d_ <- coef(m2)[2] # take the slope of m2

    V_ <- NULL												# prepare variance vector
    rep_loop <- 1:(iterations-1)		# repeat the process
    C_all <- c_												# prepare c vector
    D_all <- d_												# prepare d vector
    V_all <- V_												# prepare variance vector

    for (i in rep_loop){

      # c_ + d_ should not be in brackets as they were in a previous version
      V_ <- c_ + d_ / (total.cell.1 * spp_table$fitted_proportions * (1 - spp_table$fitted_proportions))	# work out the variance (modification of the binomial proportion variance structure)

      # take the inverse of variance squared.
      inv_sq_V <- 1 / (V_^2)

      # run the new model weighting by the inverse variance identifeid above.
      error <- try(m4 <- lm(sq_residual_m1 ~ NP_test, weights = inv_sq_V, data = spp_table), silent = TRUE)
      if(class(error) == 'try-error') stop('Model failed in iteration, too little data?')

      c_ <- coef(m4)[1] # take the intercept of the new model
      d_ <- coef(m4)[2] # take the slope of the new model

      C_all <- c(C_all,c_) # build a vector with all of the new intercepts
      D_all <- c(D_all,d_) # build a vector with all of the new slopes
      V_all <- c(V_all,V_) # build a vector with all of the new inverse variances squared

    }

    ## final model to take the residuals from
    # take the reciprocal of the "settled" variance (Telfer et al 2002)
    spp_table$recip_V <- 1 / V_
    # use the reciprocal of the "settled" variance as weight for the final model.
    m1 <- lm(T2_logit_range ~ T1_logit_range, weights = spp_table$recip_V, data = spp_table)
    # take the standardised residuals from the model.
    # For each taxa, the standardised residual from the fitted regression line provides
    # the index of relative change in range size. A taxa with a negative change index has
    # been recorded in relatively fewer grid cells in the later period, whereas a taxa with
    # a positive change index has been recorded in relatively more.
    spp_table$change_index <- rstandard(m1)
    final_output_table <- spp_table[,c(1:3,13)]

    return(list(final_output_table, spp_table))

  }

}