## Hankui wrote it on Nov 1, 2016
## confusion matrix
## source('confusion.r');

# source("getKappa.r");

confusion <- function(predicted, actual, filename="temp.csv")
{
	c_m <- as.matrix(table(Actual=actual, Predicted=predicted)) # create the confusion matrix
	# c_m <- as.data.frame(table(Predicted = predicted, Actual = actual )) # create the confusion matrix
	uniq_p <- sort(unique(predicted));
	uniq_t <- sort(unique(actual   ));
	nr      <- nrow(c_m) # number of classes

	##***************************************************************
	## important for those classes never been classified (Mar 13 2018 Hank)
	block_list <- list();
	zeros <- rowsums <- apply(c_m, 1, sum)*0;
	empty_index <- vector();
	for (i in 1:length(uniq_t)) {
		nc <- ncol(c_m);
		is_find <- 0;
		uniq_t_i <- uniq_t[i];
		for (j in 1:length(uniq_p)) if (uniq_p[j]==uniq_t_i) {is_find <- j; break; }

		if(is_find==0) {
			block_list[[i]] <- zeros;
			cat("\tclass", uniq_t_i, "is never predicted\n");
			empty_index <- c(empty_index, i);
		}  else {
			block_list[[i]] <- c_m[,is_find];
		}
	}
	c_m_new <- block_list[[1]];
	for (i in 2:length(uniq_t)) c_m_new <- cbind(c_m_new, block_list[[i]]);

	c_m <- c_m_new;
	colnames(c_m) <- rownames(c_m);

	##***************************************************************
	## accuracy
	n <- sum(c_m) # number of instances
	diags   <- diag(c_m) # number of correctly classified instances per class
	rowsums <- apply(c_m, 1, sum) # number of instances per class
	colsums <- apply(c_m, 2, sum) # number of predictions per class

	accuracy = sum(diags) / n;
	# print(c_m)
	# print(accuracy)
	# precision = diags / colsums
	# recall = diags / rowsums
	uses     <- diags / colsums * 100;
	produces <- diags / rowsums * 100;
	c_m <- cbind(c_m, rowsums, produces);

	colsumss <- c(colsums, n, 0       );
	uses.s   <- c(uses   , 0, accuracy);
	c_m <- rbind(c_m, colsumss, uses.s);

	# colnames(c_m) <- c(colnames(c_m)[1:nr],"Total truth","Producer's accuracy");
	# rownames(c_m) <- c(rownames(c_m)[1:nr],"Total predicted","User's accuracy");
	rownames(c_m) <- c(as.numeric(colnames(c_m)[1:nr])-1,"Total truth","Producer's accuracy");
	colnames(c_m) <- c(as.numeric(rownames(c_m)[1:nr])  ,"Total predicted","User's accuracy");

	# print(precision)
	options(scipen=999);
	# print(format(c_m, digits=1));
	write.csv(c_m, file=filename);

	## *******************************************************************************************
	## Kappa
	ppp <- rowsums / n; # distribution of instances over the actual classes
	qqq <- colsums / n; # distribution of instances over the predicted classes
	expAccuracy = sum(ppp*qqq);
	Kappa = (accuracy - expAccuracy) / (1 - expAccuracy);
	cat("\tKappa = ");
	cat(format(Kappa, digits=4));
	cat("\tAccuracy = ");
	print(format(accuracy*100, digits=4));
	list("Kappa"=Kappa, "Accuracy"=accuracy, "Confusion"=c_m);
}
## *******************************************************************************************
## test data
# set.seed(0)
# actual = c('a','b','c')[runif(100, 1,4)] # actual labels
# predicted = actual # predicted labels
# predicted[runif(30,1,100)] = actual[runif(30,1,100)]  # introduce incorrect predictions
# predicted[runif(30,1,10)] = c('a','b','c')[runif(10, 1,4)]  # introduce incorrect predictions
# confusion(predicted, actual);

## *******************************************************************************************
## test data global random forest
# prop <- 67;
# prop <- 100;
# load(paste("./file.plot/conus.urbanFALSE.prop",prop,".tree500", sep=""));
# actual <- truth;
# confusion(predicted, actual, "global.confusion.csv");


#  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #

#                          tttt            iiii  lllllll
#                       ttt:::t           i::::i l:::::l
#                       t:::::t            iiii  l:::::l
#                       t:::::t                  l:::::l
# uuuuuu    uuuuuuttttttt:::::ttttttt    iiiiiii  l::::l     ssssssssss
# u::::u    u::::ut:::::::::::::::::t    i:::::i  l::::l   ss::::::::::s
# u::::u    u::::ut:::::::::::::::::t     i::::i  l::::l ss:::::::::::::s
# u::::u    u::::utttttt:::::::tttttt     i::::i  l::::l s::::::ssss:::::s
# u::::u    u::::u      t:::::t           i::::i  l::::l  s:::::s  ssssss
# u::::u    u::::u      t:::::t           i::::i  l::::l    s::::::s
# u::::u    u::::u      t:::::t           i::::i  l::::l       s::::::s
# u:::::uuuu:::::u      t:::::t    tttttt i::::i  l::::l ssssss   s:::::s
# u:::::::::::::::uu    t::::::tttt:::::ti::::::il::::::ls:::::ssss::::::s
#  u:::::::::::::::u    tt::::::::::::::ti::::::il::::::ls::::::::::::::s
#   uu::::::::uu:::u      tt:::::::::::tti::::::il::::::l s:::::::::::ss
#     uuuuuuuu  uuuu        ttttttttttt  iiiiiiiillllllll  sssssssssss

#  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #

# # # backprop
# backpropagate error and update weights
#  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #
backprop <- function(out_wts, in_wts, out_activation, current_target,
                     hid_activation, hid_activation_raw, ins_w_bias, learning_rate){

  # # # calc error on output units
  out_delta <- 2 * (out_activation - current_target)

  # # # calc error on hidden units
  hid_delta <- out_delta %*% t(out_wts)
  hid_delta <- hid_delta[,2:ncol(hid_delta)] * sigmoid_grad(hid_activation_raw)

  # # # calc weight changes
  out_delta <- learning_rate * (t(hid_activation) %*% out_delta)
  hid_delta <- learning_rate * (t(ins_w_bias) %*% hid_delta)

  # # # adjust wts
  out_wts <- out_wts - out_delta
  in_wts <- in_wts - hid_delta

  return(list(out_wts = out_wts,
              in_wts  = in_wts))

}

# # # forward_pass
# conduct forward pass
#  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #
forward_pass <- function(in_wts, out_wts, inputs, out_rule) {
  # # # init needed vars
  num_feats <- ncol(out_wts)
  num_cats  <- dim(out_wts)[3]
  num_stims <- nrow(inputs)
  if (is.null(num_stims)) {num_stims <- 1}

  # # # add bias to ins
  bias_units <- matrix(rep(1, num_stims), ncol = 1, nrow = num_stims)
  ins_w_bias <- cbind(bias_units,
    matrix(inputs, nrow = num_stims, ncol = num_feats, byrow = TRUE))

  # # # ins to hids propagation
  hid_activation_raw <- ins_w_bias %*% in_wts
  hid_activation <- sigmoid(hid_activation_raw)

  # # # add bias unit to hid activation
  hid_activation <- cbind(bias_units, hid_activation)

  # # # hids to outs propagation
  out_activation <- array(rep(0, (num_stims * num_feats * num_cats)),
    dim = c(num_stims, num_feats, num_cats))

  # # NEED VECTORIZED HERE?
  # # # get output activation
  for (category in 1:num_cats) {
  	out_activation[,,category] <- hid_activation %*% out_wts[,,category]
  }

  # # # apply output activatio rule
  if(out_rule == 'sigmoid') {
  	out_activation <- sigmoid(out_activation)
  }

  return(list(out_activation     = out_activation,
              hid_activation     = hid_activation,
              hid_activation_raw = hid_activation_raw,
              ins_w_bias         = ins_w_bias))

}

# # # get_wts
# generate net weights
#  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #
get_wts <- function(num_feats, num_hids, num_cats, wts_range, wts_center) {
  # # # set bias
  bias <- 1

  # # # generate wts between ins and hids
  in_wts <-
    (matrix(runif((num_feats + bias) * num_hids), ncol = num_hids) - 0.5) * 2
  in_wts <- wts_center + (wts_range * in_wts)

  # # # generate wts between hids and outs
  out_wts <-
    (array(runif((num_hids + bias) * num_feats * num_cats),
      dim = c((num_hids + bias), num_feats, num_cats)) - 0.5) * 2
  out_wts <- wts_center + (wts_range * out_wts)

  return(list(in_wts  = in_wts,
              out_wts = out_wts))

}

# # # global_scale
# scale inputs to 0/1
#  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #
global_scale <- function(x) { x / 2 + 0.5 }

# # # train_plot
# function to produce line plot of training
#  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #
train_plot <- function(training) {
  # # # get dimensions
  n_cats <- dim(training)[2]
  xrange <- c(1, dim(training)[1])
  yrange <- range(training)

  # # # open plot device
  pdf('training_plot.pdf')

  # # # create frame
  plot(xrange, c(.01, yrange[2]), xlab = 'Block Number', ylab = 'Accuracy')

  # # # aesthetics
  colors <- rainbow(n_cats)
  line_type <- c(1:n_cats)
  plot_char <- seq(18, 18 + n_cats, 1)

  # # # plot lines
  for (i in 1:n_cats) {
    target_cat <- training[,i]
    lines(seq(1, xrange[2], 1), target_cat, type = 'b', lwd = 1.5,
      lty = line_type[i], col = colors[i],  pch = plot_char[i])
  }

  # # # title and legend
  title('DIVA Training Accuracy across Blocks')
  legend('bottomright', y = NULL, 1:n_cats, cex = 0.8, col = colors,
    pch = plot_char, lty = line_type, title = 'SHJ Categories')

  # # # produce plot
  dev.off()

}

# response_rule
# convert output activations to classification
#  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #
response_rule <- function(out_activation, target_activation, beta_val){
  num_feats <- ncol(out_activation)
  num_cats  <- dim(out_activation)[3]
  num_stims <- nrow(target_activation)
  if (is.null(num_stims)) {num_stims <- 1}

  # # # calc error
  ssqerror <- array(as.vector(
    apply(out_activation, 3, function(x) {x - target_activation})),
      c(num_stims, num_feats, num_cats))
  ssqerror <- ssqerror ^ 2
  ssqerror[ssqerror < 1e-7] <- 1e-7

  # # # if focusing is on:
  if (beta_val > 0) {
    # # # get list of channel comparisons
    pairwise_comps <- combn(1:num_cats, 2)

    # # # get differences for each feature between categories
    diff_matrix <-
      abs(apply(pairwise_comps, 2, function(x) {
        out_activation[,,x[1]] - out_activation[,,x[2]]}))

    # # # reconstruct activation array and get feature diversity means
    diff_array <- array(diff_matrix, dim = c(num_stims, num_feats, num_cats))
    feature_diffs <- apply(diff_array, 2, mean)

    # # # calculate diversities
    diversities <- exp(beta_val * feature_diffs)
    diversities[diversities > 1e+7] <- 1e+7

    # # # divide diversities by sum of diversities
    fweights = diversities / sum(diversities)

    # # # apply focus weights; then get sum for each category
    ssqerror <- t(apply(ssqerror, 3, function(x) sum(x * fweights)))

  # # # otherwise, set focus weights to NULL
  } else {
    fweights <- NULL
  }

  # # # calculate inverse sse
  ssqerror <- 1 / ssqerror

return(list(ps       = (ssqerror / sum(ssqerror)),
            fweights = fweights,
            ssqerror = ssqerror))

}

# run_diva
# trains vanilla diva
#  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #
run_diva <- function(model) {

  # # # get new seed
  model.seed <- runif(1) * 100000 * runif(1)
  set.seed(model.seed)

  # # # set mean value of weights
  model$wts_center <- 0
  # # # convert targets to 0/1
  model$targets <- global_scale(model$inputs)
  # # # init size parameter variables
  model$num_feats   <- ncol(model$inputs)
  model$num_stims   <- nrow(model$inputs)
  model$num_cats    <- length(unique(model$labels))
  model$num_updates <- model$num_blocks * model$num_stims
  # # # init training accuracy matrix
  training <-
    matrix(rep(NA, model$num_updates * model$num_inits),
      nrow = model$num_updates, ncol = model$num_inits)

  # # # initialize and run DIVA models
  for (model_num in 1:model$num_inits) {

    # # # generate weights
    wts <- get_wts(model$num_feats, model$num_hids, model$num_cats,
      model$wts_range, model$wts_center)

    # # # generate random presentation order
    prez_order <- as.vector(apply(replicate(model$num_blocks,
      seq(1, model$num_stims)), 2, sample, model$num_stims))

    # # # iterate over each trial in the presentation order
    for (trial_num in 1:model$num_updates) {
      current_input  <- model$inputs[prez_order[[trial_num]], ]
      current_target <- model$targets[prez_order[[trial_num]], ]
      current_class  <- model$labels[prez_order[[trial_num]]]

      # # # complete forward pass
      fp <- forward_pass(wts$in_wts, wts$out_wts, current_input, model$out_rule)

      # # # calculate classification probability
      response <- response_rule(fp$out_activation, current_target, model$beta_val)

      # # # store classification accuracy
      training[trial_num, model_num] = response$ps[current_class]

      # # # back propagate error to adjust weights
      class_wts <- wts$out_wts[,,current_class]
      class_activation <- fp$out_activation[,,current_class]

      adjusted_wts <- backprop(class_wts, wts$in_wts, class_activation, current_target,
               fp$hid_activation, fp$hid_activation_raw, fp$ins_w_bias, model$learning_rate)

      # # # set new weights
      wts$out_wts[,,current_class] <- adjusted_wts$out_wts
      wts$in_wts <- adjusted_wts$in_wts

    }

  }

training_means <-
  rowMeans(matrix(rowMeans(training), nrow = model$num_blocks, ncol = model$num_stims, byrow = TRUE))

return(list(training = training_means,
            model    = model))

}

# shj_cats
# loads shj category structures
#  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #
demo_cats <- function(type){

  in_pattern <-
    matrix(c(-1, -1, -1,
	           -1, -1,  1,
	           -1,  1, -1,
  	         -1,  1,  1,
	            1, -1, -1,
	            1, -1,  1,
	            1,  1, -1,
	            1,  1,  1),
      nrow = 8, ncol = 3, byrow = TRUE)

  cat_assignment <-
    matrix(c(1, 1, 1, 1, 2, 2, 2, 2,  # type I
             1, 1, 2, 2, 2, 2, 1, 1,  # type II
             1, 1, 2, 1, 1, 2, 2, 2,  # type III
             1, 1, 1, 2, 1, 2, 2, 2,  # type IV
             2, 1, 1, 1, 1, 2, 2, 2,  # type V
             1, 2, 2, 1, 2, 1, 1, 2,  # type VI
             1, 1, 2, 2, 3, 3, 4, 4), # type II multiclass
      ncol = 8, byrow = TRUE)

return(list(inputs = in_pattern,
			      labels = cat_assignment[type,]))

}

# sigmoid
# returns sigmoid evaluated elementwize in X
#  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #
sigmoid <- function(x) {
  g = 1 / (1 + exp(-x))

return(g)

}

# sigmoid gradient
# returns the gradient of the sigmoid function evaluated at x
#  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #  #
sigmoid_grad <- function(x) {

return(g = ((sigmoid(x)) * (1 - sigmoid(x))))

}


## Load some packages that will be useful
install.packages("mgcv") # load the MGCV package with Simon Wood's gam() functions
install.packages("readr") # Hadley Wickham's package for reading in data easily
install.packages("ggplot2") # for plotting with ggplot
install.packages("ggfortify")
library(mgcv) # load the MGCV package with Simon Wood's gam() functions
library(readr) # Hadley Wickham's package for reading in data easily
library(ggplot2) # for plotting with ggplot
library(ggfortify) # for autoplot

##################################################################################################
## We'll first examine the bias-variance tradeoff in a simplified form with a small exercise.
## If you remember, this tradeoff is one of the concepts behind finding the optimal amount of smoothing.
## We'll fit linear regressions to parts of the data, increasing the number of parts from 1 to 10.
##################################################################################################

# First, read in the bioluminescence data.
# Sources is the number of bioluminescent plankton seen in each water sample
# SampleDepth is the depth of the water sample, in m
# Station is the ID of the location at which multiple water samples were taken
ISIT <- read_tsv(url('https://github.com/aeda2021/2021_master/raw/main/raw-data/ISIT.txt')) # tsv for tab-separated values

# Subset it to one station. This is what we'll work with for now.
ISITsub <- subset(ISIT, Station == 8)

# Plot the data
ggplot(ISITsub, aes(x=SampleDepth, y=Sources)) +
	geom_point()

# Now you can fit a linear regression to the ISITsub dataset, with Sources as the response and SampleDepth as the explanatory variable
# Please call the output "mod1", as I've started to fill in for you
mod1 <- lm (ISITsub$Sources ~ ISITsub$SampleDepth, data = ISITsub)

# Q1: What was your code for fitting a linear regression of Sources vs. SampleDepth?
#see above

# We'll predict the mean response and confidence intervals from this model and save the output
ISITsub$mod1 <- predict(mod1)
ISITsub$mod1lwr <- predict(mod1, interval='confidence', level=0.95)[,'lwr']
ISITsub$mod1upr <- predict(mod1, interval='confidence', level=0.95)[,'upr']

# Now plot the linear model fit to the full dataset on top of the data
ggplot(ISITsub, aes(x=SampleDepth, y=Sources)) +
	geom_point() +
	geom_line(aes(y=mod1)) +
	geom_ribbon(aes(ymin=mod1lwr, ymax=mod1upr), alpha=0.3)

# Q2: Use your skills from earlier in this course and evaluate this linear model (autoplot may be useful here). How does this fit look to you? Does a linear regression look reasonable? Any well-justified answer is appropriate.
autoplot(mod1)
#we want residuals around 0, so the residuals are not very normal; therefore a linear regression is not reasonable.

# For comparison, let's fit linear models to two halves of the data. This involves a fair bit of coding in R, so please ask for help understanding the code where things are confusing.
# First, we have to find the halves of the data
quants2 <- quantile(ISITsub$SampleDepth, probs=seq(0, 1, by=0.5)) # define the ends and the half-way point in the data with the quantile function
ISITsub$split2 <- cut(ISITsub$SampleDepth, breaks=quants2, include.lowest=TRUE, labels=1:2)

# Q3: In your own words, what does the cut() function do? How are we using it here?
#The cut function allows us to split up data into two lables
#here we are using it to split the data along the x axis into two subsets (labeled 1 and 2) based on where they fall (first half or second)

# Next, let's fit a linear model to each subset (half) of the data. We do this in a loop to make it easier.
mod2s <- vector('list', length=2) # A list to hold each of our models in a single, convenient object
ISITsub$mod2 <- ISITsub$mod2lwr <- ISITsub$mod2upr <- NA # Initialize the variables we'll use and fill with NA
for(i in 1:length(mod2s)){ # Loop through each model
	mod2s[[i]] <- lm(Sources ~ SampleDepth, data=ISITsub[ISITsub$split2==i,]) # Fit a linear regression to the appropriate subset of the data. Now see why we made the split2 vector?
	ISITsub$mod2[ISITsub$split2==i] <- predict(mod2s[[i]]) # Now predict from that model. Notice we're only predicting for part of the data.
	ISITsub$mod2lwr[ISITsub$split2==i] <- predict(mod2s[[i]], interval='confidence', level=0.95)[,'lwr'] # Same for the confidence intervals
	ISITsub$mod2upr[ISITsub$split2==i] <- predict(mod2s[[i]], interval='confidence', level=0.95)[,'upr']
}

# Plot the data, the 1-model fit, and the 2-model fit
# There will be an odd 3rd line segment connecting the end of the first model fit to the beginning of the second model fit. Just pretend it's not there.
ggplot(ISITsub, aes(x=SampleDepth, y=Sources)) +
	geom_point() +
	geom_line(aes(y=mod1)) +
	geom_ribbon(aes(ymin=mod1lwr, ymax=mod1upr), alpha=0.3) +
	geom_line(aes(y=mod2, color='red')) +
	geom_ribbon(aes(ymin=mod2lwr, ymax=mod2upr), alpha=0.3, fill='red')

# Q4: Has the fit between the model and the data improved? What's the downside of continuing the split the data into finer and finer chunks and fitting more models?
#Yes, the fit has improved! However, the downside of continuing to split the data/fit more models is the trap of overfitting
#you're not modelling the general trend, you'll eventually just show the exact data set

# Now, it's your turn. Please fit 10 linear regression models. I'd recommend using the 2-model code above as a guide and tweaking it.

quants10 <- quantile(ISITsub$SampleDepth, probs=seq(0, 1, by=0.1)) # define the ends and the 10 points
ISITsub$split10 <- cut(ISITsub$SampleDepth, breaks=quants10, include.lowest=TRUE, labels=1:10)
mod10 <- vector('list', length=10) # A list to hold each of our models in a single, convenient object
ISITsub$mod10 <- ISITsub$mod10lwr <- ISITsub$mod10upr <- NA # Initialize the variables we'll use and fill with NA
for(i in 1:length(mod10)){ # Loop through each model
    mod10[[i]] <- lm(Sources ~ SampleDepth, data=ISITsub[ISITsub$split10==i,]) # Fit a linear regression to the appropriate subset of the data. Now see why we made the split2 vector?
    ISITsub$mod10[ISITsub$split10==i] <- predict(mod10[[i]]) # Now predict from that model. Notice we're only predicting for part of the data.
    ISITsub$mod10lwr[ISITsub$split10==i] <- predict(mod10[[i]], interval='confidence', level=0.95)[,'lwr'] # Same for the confidence intervals
    ISITsub$mod10upr[ISITsub$split10==i] <- predict(mod10[[i]], interval='confidence', level=0.95)[,'upr']
}

# Now plot the 10 models on top of the data and on top of the 1- and 2-model fits. The plot with the data, 1-model, and 2-model fit is probably a useful guide here.

ggplot(ISITsub, aes(x=SampleDepth, y=Sources)) +
    geom_point() +
    geom_line(aes(y=mod10)) +
    geom_ribbon(aes(ymin=mod10lwr, ymax=mod10upr), alpha=0.3) +
    geom_line(aes(y=mod10, color='red')) +
    geom_ribbon(aes(ymin=mod10lwr, ymax=mod10upr), alpha=0.3, fill='red')

# Q5: Compare your 10-model fit to the 1-model fit. Which model's predictions are furthest from the observed data? Which model has the widest confidence bounds? How does this (or does this not) illustrate the bias-variance tradeoff?
#Model 1 was the furthest from observed data, but Model 10 had the widest confidence bounds
#bias-variance tradeoff = increasing bias to reduce variance
#mod1 is biased because we assume a linear fit; mod10 has high variance, but it fits so well

####################################################
## Now you'll fit some GAMs and evaluate your models
####################################################
#install.packages("maps") # if needed
library(maps) # has map data in it

# load the data
spdata <- readRDS(url('https://github.com/aeda2021/2021_master/raw/main/raw-data/gadusmorhua.rds')) # load the spdata data.frame with abundance and environmental data from cod surveys
spdata$presfit01 <- as.numeric(spdata$presfit) # make a vector that's nice for plotting
spdata <- spdata[order(spdata$presfit01),] # order the data from absent to present for ease of plotting

# examine the data
head(spdata) # look at the dataset
summary(spdata)

# make a map of the data
world <- map_data('world')

ggplot() +
	geom_polygon(data=world, aes(x=long, y=lat, group=group), color='black', fill=NA) + # the map
	xlim(-100, -45) +
	ylim(23, 62) +
	geom_point(data=spdata, aes(x=lon, y=lat, color=presfit, alpha=0.1), size=0.01) # the data points. note that because the dataframe is sorted from absent to present, the presences are plotted on top of the absences

# Q6: Based on the map, do you think cod prefer warmer or cooler waters? Why do you think that?
#Cod like cooler water. I think that because all of the blue points are in the nothern area = colder water

# Q7: Plot cod presence/absence (the presfit vector) vs. the bottom temperature (SBT.actual). At what range of temperatures have cod been observed?

ggplot(spdata, aes(x=SBT.actual, y=presfit01)) +
    geom_point()
#cod can be found in the range ~0 - 20 Celsius

# Q8: Fit a GAM for presfit against SBT.actual. Check the model. Does it look like the assumptions have been met? Why or why not?

gam1 <- gam(presfit01 ~ s(SBT.actual, k=8), data=spdata)
gam.check(gam1)

#does not meet assumptions, using plots we can see that it is violating normality

# Q9: Fit a new GAM with a more appropriate error structure and save it as "mod2". The gam() function takes the same family= argument as does glm(). Which error structure and link function did you choose?
mod2 <- gam(presfit01 ~ s(SBT.actual, k=8), data=spdata, family=binomial)
gam.check(mod2)

# Q10: Check your mod2 GAM. Does it look like the assumptions have been met? Make sure to read the text output as well as look at the graphs. If something doesn't look right, what would you do to fix it?

#yes, the deviance residuals vs. theoretical quantiles plot follows the 1:1 ratio
#I changed the k value because the output showed a low p-value, indicating that the edf and k value were too close

# Q11: Interpret your mod2 GAM (summary function). How much deviance is explained? Is the SBT.actual term significant at alpha=0.05? How wiggly do you expect the smooth fit to be?
#deviance = 25.4%
#significance = yes, the SBT.actual is significant
#line = pretty curvy, since edf is a 9
summary(mod2)

# Now let's make predictions from our last model (mod2) and plot them.
nd <- data.frame(SBT.actual = seq(-5,30, by=0.5)) # make a data.frame of new explanatory variables
nd$mod2 <- predict(mod2, newdata=nd, type='response') # predict on the scale of the response
nd$mod2se <- predict(mod2, newdata=nd, type='response', se.fit=TRUE)$se.fit # get the standard errors of the fit

ggplot(spdata, aes(x=SBT.actual, y=presfit01)) +
	geom_point() +
	geom_line(data=nd, aes(x=SBT.actual, y=mod2, color='red')) +
	geom_ribbon(data=nd, aes(x=SBT.actual, ymin=mod2-mod2se, ymax=mod2+mod2se), alpha=0.3, fill='red', inherit.aes=FALSE)

# Q12: Does the fit look realistic? Why or why not? Does it look overfit? Why or why not?
#I feel like the fit is not realistc because it has many curves; it looks overfit
#it captures too many of the ups and downs to be realistic

# Q13: This dataset is actually a concatenation of seven different surveys which each have slightly different abilities to catch cod. Fit a new GAM that includes a categorical predictor for survey (the region vector). Based on AIC, which model would you choose? How confident would you be?
mod3 <- gam(presfit01 ~ s(SBT.actual, k=6) + region, data=spdata, family=binomial)
library(MuMIn) # for model selection
model.sel(mod3, mod2)

#AIC shows that model 3, which incorporates region, is the better model. Pretty confident because it is quite a big difference between models.


#' Principal Components Analysis
#'
#' Performs the principal components analysis.
#'
#' @details
#' \code{prcomp()} performs the principal components analysis on the data
#' matrix by taking the SVD. Sometimes core R and kazaam will disagree
#' slightly in what the rotated variables are because of how the SVD is
#' caluclated.
#'
#' @section Communication:
#' The communication is an \code{allreduce()} call, quadratic on the number of
#' columns.  Most of the run time should be dominated by relatively expensive
#'  local operations.
#'
#' @param x
#' A shaq.
#' @param center
#' Should columns are zero centered?
#' @param scale.
#' Should columns are rescaled to unit variance?
#' @param retx
#' Should the rotated variables be returned?
#' @param tol
#' The cutoff for components.
#' @param ...
#' Ignored.
#'
#' @return
#' A list of elements \code{sdev}, \code{rotation}, \code{center}, \code{scale},
#' and \code{x}, as with R's own \code{prcomp()}.  The elements are,
#' respectively, a regular vector, a regular matrix, a regular vector, a regular
#' vector, and a shaq.
#'
#' @examples
#' \dontrun{
#' library(kazaam)
#' x = ranshaq(runif, 10, 3)
#' pca = prcomp(x)
#'
#' comm.print(pca)
#'
#' finalize()
#' }
#'
#' @name prcomp
#' @rdname prcomp
#' @export
prcomp.shaq = function(x, retx=TRUE, center=TRUE, scale.=FALSE, tol=NULL, ...)
{
  x <- scale(x, center=center, scale=scale.)
  x.center <- attr(DATA(x), "scaled:center")
  x.scale <- attr(DATA(x) , "scaled:scale")
  if (any(x.scale == 0))
      comm.stop("cannot rescale a constant/zero column to unit variance")

  s <- svd(x, nu=0)
  s$d <- s$d/sqrt(max(1, nrow(x) - 1))

  if (!is.null(tol))
  {
    rank <- max(as.integer(sum(s$d > (s$d[1L] * tol))), 1L)
    if (rank < ncol(x))
    {
      s$v <- s$v[, 1L:rank, drop=FALSE]
      s$d <- s$d[1L:rank]
    }
  }

  if (is.null(x.center))
    center = FALSE
  else
    center = x.center

  if (is.null(x.scale))
    scale = FALSE
  else
    scale = x.scale

  r <- list(sdev=s$d, rotation=s$v, center=center, scale=scale)
  if (retx)
    r$x <- x %*% s$v

  class(r) <- "prcomp"

  return(r)
}


#'@title calcHMAppendageRes
#'
#'@description Calculate appendage resistance (\code{Rapp}) (kN) from the
#'Holtrop & Mennen method.
#'
#'@param shipSpeed Ship actual speed (vector of numericals, m/s) (see
#' \code{\link{calcSpeedUnitConversion}})
#'@param Cf Frictional resistance coefficient (vector of numericals,
#' dimensionless) (see \code{\link{calcCf}})
#'@param appendagesList List of appendages on ship (vector of strings) \itemize{
#'\item"rudder behind skeg"
#'\item"rudder behind stern"
#'\item"twin-screw balance rudders"
#'\item"shaft brackets"
#'\item"skeg"
#'\item"strut bossings"
#'\item"hull bossings"
#'\item"shafts"
#'\item"stabilizer fins"
#'\item"dome"
#'\item"bilge keels"
#'}
#'@param wettedAppSAList List of wetted surface areas corresponding to list of appendages
#' (vector of numericals, m^2)
#'@param seawaterDensity Sea water density. Default = 1.025 (g/cm^3). Can
#' supply either a vector of numericals corresponding to the ship speed, provide a single
#' number, or rely on the default
#'
#'@return \code{Rapp} (vector of numericals, kN)
#'
#'@references
#'Holtrop, J. and Mennen, G. G. J. 1982. "An approximate power prediction
#'method." International Shipbuilding Progress 29.
#'
#'@seealso \itemize{
#'\item \code{\link{calcSpeedUnitConversion}}
#'\item \code{\link{calcCf}} }
#'
#'@family Holtrop-Mennen Calculations
#'@family Resistance Calculations
#'
#'@examples calcHMAppendageRes(seq(1,5,1),0.0015,"rudder behind skeg",50,seawaterDensity=1.025)
#' calcHMAppendageRes(seq(1,5,1),0.0015,NA,0,seawaterDensity=1.025)
#'
#'@export

calcHMAppendageRes<- function(shipSpeed,Cf,
       appendagesList,wettedAppSAList,seawaterDensity=1.025){

AppResFactor <- data.frame(appendage=c("rudder behind skeg","rudder behind stern",
"twin-screw balance rudders","shaft brackets","skeg","strut bossings","hull bossings",
"shafts","stabilizer fins","dome","bilge keels"),
resistance.factor=c(1.5,1.4,2.8,3,1.75,3,2,3,2.8,2.7,1.4))

wettedAppSAList[is.na(wettedAppSAList)==TRUE]<-0
appendagesList<- tolower(appendagesList)

FormFactorEqu=0

for(i in 1:length(appendagesList)){
  if(length(AppResFactor$resistance.factor[AppResFactor$appendage==paste(appendagesList[i])]*
      wettedAppSAList[i])>0){
 j<- AppResFactor$resistance.factor[grepl(appendagesList[i],AppResFactor$appendage)==TRUE]*
   wettedAppSAList[i]}else{j<-0}
  FormFactorEqu=FormFactorEqu+j
}

FormFactorEqu<-FormFactorEqu/sum(wettedAppSAList)
FormFactorEqu[is.na(FormFactorEqu)]<-0
Rapp<-0.5*seawaterDensity*(shipSpeed^2)*wettedAppSAList*FormFactorEqu*Cf

return(Rapp)
}


# Current method

# TODO Consider implementing this from scratch so that you know the details
# It should be straightforward, see here: https://bookdown.org/egarpor/NP-UC3M/kre-i-kre.html
# Using kernsmooth currently because I expect their implementation is faster and more robust

# The following functions contains Kernsmooth and ROT estimators.  ROT is selected currently

# Returns Lambda, or P(+|S=t)
EstLambda = function(S, X, t, idx, h = NULL){

  tryCatch({
    Sorder <- order(S, decreasing=TRUE)
    m <- length(S)
    idx.range <- c(max(1, idx - 1000), min(length(S), idx + 1000))

    S.ordered <- S[Sorder][idx.range[1]:idx.range[2]]
    X.ordered <- X[Sorder][idx.range[1]:idx.range[2]]

    ## --1 Kernsmooth bandwidth estimator
    ## more accurate, but prone to errors
    # if(is.null(h)) h <- KernSmooth::dpill(x = S.ordered, y = X.ordered, gridsize = 100)

    ## --2 Rule of thumb estimator
    ## Should not throw an error
    if(is.null(h)) h <- (m^{-1/5}) * sd(S)

    lp0 <- KernSmooth::locpoly(x = S.ordered, y = X.ordered, bandwidth = h, degree = 0,
                               range.x = range(S.ordered), gridsize = 1000)

    lp0.idx <- which.min(abs(lp0$x - t))
    lam <- lp0$y[lp0.idx]
  }, error = function(e) {
    stop("Lambda estimate failed.  Try setting h manually.")
  }
  )

  return(lam)
}

# --3 JHO
#
# # Returns Lambda, or P(+|S=t)
# EstLambda = function(S, X, t, ...){
#  #h is the bandwidth, t is called c in the JZ paper
#  #m is the sample size, S and X are the vectors of scores and labels
#  m <- length(S)
#  h <- (m^{-1/5}) * sd(S)
#  ind.win <- (S < t + h) & (S > t - h)
#  exp.X.and.I <- sum(X*ind.win)/m
#  exp.I <- sum(ind.win)/m
#  return(exp.X.and.I/exp.I)
# }

# --4 np method

# # Returns Lambda, or P(+|S=t)
# EstLambda = function(S, X, t, idx, Sorder, h = NULL){
#
#   idx.range <- c(max(1, idx - 100), min(length(S), idx + 100))
#
#   S.ordered <- S[Sorder][idx.range[1]:idx.range[2]]
#   X.ordered <- X[Sorder][idx.range[1]:idx.range[2]]
#
#   # TODO This is still not silent!
#   if(is.null(h)) {
#     bwa <- invisible(np::npregbw(formula = X.ordered ~ S.ordered, bwtype = "adaptive_nn",
#                                  regtype = "lc"))
#   } else {
#     bwa <- invisible(np::npregbw(formula = X.ordered ~ S.ordered, bws = h,
#                                  regtype = "lc"))
#   }
#   np.model <- invisible(np::npreg(bwa))
#
#   return(predict(np.model, newdata = data.frame(S.ordered = t)))
# }

# --5 JZ original method
#
# Returns Lambda, or P(+|S=t)
# EstLambda = function(S, X, t, ...){
#  #h is the bandwidth, t is called c in the JZ paper
#  #m is the sample size, S and X are the vectors of scores and labels
#  m <- length(S)
#  h <- (m^{-1/5})
#  ind.win <- (S < t + h) & (S > t - h)
#  exp.X.and.I <- sum(X*ind.win)/m
#  exp.I <- sum(ind.win)/m
#  return(exp.X.and.I/exp.I)
# }

# Bootstrap percentile CI
BootCI = function(X, S, m, pi.0, boot.rep, metric, plus, r, myseed=111){
  storeout=matrix(NA, nrow=m, ncol=1+boot.rep)
  r.all <- (1:m)/m
  idx <- which(r.all %in% r)
  storeout <- storeout[idx, ]
  storeout[,1] <- r
  set.seed(myseed)
  for(i in 1:boot.rep){
    boot.samp <- sample(1:m, m, replace = T)
    X.star = X[boot.samp]
    S.star = S[boot.samp]
    Sorder <-  order(S.star,decreasing=TRUE)
    hits <- cumsum(X.star[Sorder])[idx]
    pi.0 <- mean(X.star)
    pi <- (hits)/(m*r)
    k <- (hits)/(sum(X.star))
    if (plus) {
      # using random plus correction
      plus.yes <- rbinom(1, 4, .5)
      pi <- (hits+plus.yes)/(m*r+4)
      k <- r/pi.0*pi
    }
    if(metric == "rec") {
      storeout[,1+i] = k
    } else if(metric == "prec") {
      storeout[,1+i] = pi
    } else if(metric == "lift") {
      storeout[,1+i] = k/r
    }
    # print(i/boot.rep)
  }
  A = apply(storeout[,-1],MARGIN=1,FUN=function(x){quantile(x, probs = c(.025, .975))})
  # Returns 95% percentile bootstrap intervals at quantiles
  A=t(A)
}

#' Construct a confidence band for a recall or precision curve
#'
#' \code{PerfCurveBands} takes a pair of score and activity vectors as input.
#' A performance curve and confidence band is created for the selected testing fractions.
#'
#' @param S a vector of scores.
#' @param X a vector of activities.
#' @param r a vector of testing fractions.
#' @param metric the performance curve to use. Options are recall ("rec") and precision ("prec").
#' @param type specifies whether a point-wise confidence interval
#' ("pointwise") or a confidence band ("band") should be constructed.
#' @param method the method to use. Point-wise confidence interval options
#' are "binomial", "JZ", "bootstrap". Confidence band options are "sup-t", "theta-proj".
#' @param plus should plus correction be used or not?
#' @param conf.level the confidence level for the bands.
#' @param boot.rep the number of replicates to use for the bootstrap method.
#' @param mc.rep the number of Monte Carlo replicates to use for the sup-t method.
#' @param myseed the random seed.
#'
#' @export
PerfCurveBands <- function(S, X, r, metric = "rec", type = "band", method = "sup-t",
                           plus = T, conf.level = .95, boot.rep = 100,
                           mc.rep = 100000, myseed = 111, h = NULL){

  # TODO add support for EF

  # Some error handeling

  if (!(method %in% c("binomial", "JZ", "bootstrap", "sup-t", "theta-proj", "bonf"))) {
    stop("'method' should be a string specifiying a performance curve method in chemmodlab.
         Point-wise confidence interval options are 'binomial', 'JZ', 'bootstrap'. Confidence band options
         are 'sup-t', 'theta-proj'.")
  }

  if (!(metric %in% c("rec", "prec"))) {
    stop("'metric' should be a string specifiying a performance curve metric in chemmodlab.
         Point-wise confidence interval options are 'binomial', 'JZ', 'bootstrap'. Options
         are recall ('rec') and precision ('prec').")
  }

  if (!(type %in% c("band", "pointwise"))) {
    stop("'type' should be a string specifiying a performance curve type in chemmodlab.
         Point-wise confidence interval options are 'binomial', 'JZ', 'bootstrap'. Options
         are point-wise confidence interval ('pointwise') or a confidence band ('band').")
  }

  set.seed(myseed)
  alpha <- 1-conf.level
  m <- length(S) #total sample size
  nact <- sum(X)  #total number of actives
  ntest <- m*r #total number tested
  r.all <- (1:m)/m
  idx <- which(r.all %in% r)

  # Convert fractions in r to thresholds, for both sets of scores
  # Also accumulate the number of actives, for each score and jointly
  Fcdf <- ecdf(S); yyobs <- sort(unique(S))
  Finv <- stepfun(x = Fcdf(yyobs), y = c(yyobs, max(yyobs)), right=TRUE, f=1)
  hits <- vector(length = length(r))
  t <- vector(length = length(r))
  for(i in 1:length(r)) {
    t[i] <- Finv(1-r[i])
    hits[i] <- sum(X*(S > t[i]))
  }

  Lam.vec <- vector(length = length(idx))
  for(j in seq_along(idx)){
    Lam.vec[j] <- EstLambda(S, X, t = t[j], idx = idx[j], h)
  }

  pi <- (hits)/(m*r)
  k <- hits/nact

  # Plus 2 correction
  if (plus) {
    hits <- hits + 2
    nact <- nact + 4
    ntest <- ntest + 2
    m <- m + 4
  }

  pi.0 <- nact/m
  r <- ntest/m
  k.c <- hits/nact
  pi.c <- pi.0/r*k.c
  k.ide <- (cumsum(rev(sort(X)))[idx]/sum(X))

  CI.int <- matrix(ncol = 2, nrow = length(k))
  if(type == "pointwise") {

    # Z Quantile for CIs
    quant <- qnorm(1-alpha/2)

    if(metric == "rec") {
      if(method == "JZ"){
        for(j in seq_along(k)) {
          Lam <- Lam.vec[j]
          var.k <- ((k.c[j]*(1-k.c[j]))/(m*pi.0))*(1-2*Lam) +
            (Lam^2*(1-r[j])*r[j])/(m*pi.0^2)
          # Check to see if var.k is negative due to machine precision problem
          var.k <- ifelse(var.k < 0, 0, var.k)
          sd.k <- sqrt(var.k)
          lcl <- k.c[j] - quant*sd.k
          lcl <- ifelse(lcl < 0, 0, lcl)
          ucl <- k.c[j] + quant*sd.k
          ucl <- ifelse(ucl > k.ide[j], k.ide[j], ucl)
          CI.int[j, ] <- c(lcl, ucl)
        }
      } else if(method == "bootstrap") {
        # bootstrap quantiles
        CI.int <- BootCI(X, S, m, pi.0, boot.rep, metric = "rec", plus, r, myseed=myseed)
        CI.int[, 1] <- ifelse(CI.int[, 1] < 0, 0, CI.int[, 1])
        CI.int[, 2] <- ifelse(CI.int[, 2] > k.ide, k.ide, CI.int[, 2])
      } else if(method == "binomial") {
        for(j in seq_along(k)) {
          var.k <- ((m*pi.0)^-1)*k.c[j]*(1-k.c[j])
          var.k <- ifelse(var.k < 0, 0, var.k)
          sd.k <- sqrt(var.k)
          lcl <- k.c[j] - quant*sd.k
          lcl <- ifelse(lcl < 0, 0, lcl)
          ucl <- k.c[j] + quant*sd.k
          # TODO consider removing due to coverage issues
          ucl <- ifelse(ucl > k.ide[j], k.ide[j], ucl)
          CI.int[j, ] <- c(lcl, ucl)
        }
      }
    } else if(metric == "prec") {
      if(method == "JZ") {
        for(j in seq_along(k)) {
          Lam <- Lam.vec[j]
          var.pi <- (pi.c[j]*(1-pi.c[j]))/(m*r[j]) +
            (1-r[j])*(pi.c[j]-Lam)^2/(m*r[j])
          # Check to see if var.pi is negative due to machine precision issues
          var.pi <- ifelse(var.pi < 0, 0, var.pi)
          sd.pi <- sqrt(var.pi)
          lcl <- pi.c[j] - quant*sd.pi
          lcl <- ifelse(lcl < 0, 0, lcl)
          ucl <- pi.c[j] + quant*sd.pi
          ucl <- ifelse(ucl > 1, 1, ucl)
          CI.int[j, ] <- c(lcl, ucl)
        }
      } else if(method == "bootstrap") {
        # bootstrap quantiles
        CI.int <- BootCI(X, S, m, pi.0, boot.rep, metric = "prec", plus, r, myseed=myseed)
        CI.int[, 1] <- ifelse(CI.int[, 1] < 0, 0, CI.int[, 1])
        CI.int[, 2] <- ifelse(CI.int[, 2] > 1, 1, CI.int[, 2])
      } else if(method == "binomial") {
        for(j in seq_along(pi)) {
          var.pi <- ((m*r[j])^-1)*pi.c[j]*(1-pi.c[j])
          # Check to see if var.pi is negative due to machine precision issues
          var.pi <- ifelse(var.pi < 0, 0, var.pi)
          sd.pi <- sqrt(var.pi)
          lcl <- pi.c[j] - quant*sd.pi
          lcl <- ifelse(lcl < 0, 0, lcl)
          ucl <- pi.c[j] + quant*sd.pi
          ucl <- ifelse(ucl > 1, 1, ucl)
          CI.int[j, ] <- c(lcl, ucl)
        }
      }
    }
  } else if(type == "band") {
    if(metric == "rec") {
      if(method == "sup-t") {
        cor.C <- matrix(NA, ncol = length(k), nrow = length(k))
        for(f in seq_along(k)) {
          for(e in 1:f) {
            Lam1 <- Lam.vec[e]
            var.k1 <- (((k.c[e]*(1-k.c[e]))/(m*pi.0))*(1-2*Lam1) +
                         (Lam1^2*(1-r[e])*r[e])/(m*pi.0^2))
            var.k1 <- ifelse(var.k1 < 0, 0, var.k1)
            Lam2 <- Lam.vec[f]
            var.k2 <- (((k.c[f]*(1-k.c[f]))/(m*pi.0))*(1-2*Lam2) +
                         (Lam2^2*(1-r[f])*r[f])/(m*pi.0^2))
            var.k2 <- ifelse(var.k2 < 0, 0, var.k2)
            cov.k <- ((m^-1*pi.0^-2)*(pi.0*(k.c[e]-k.c[e]*k.c[f])*(1-Lam1-Lam2) +
                                        (r[e]-r[e]*r[f])*Lam1*Lam2))
            if(e == f) {
              # If the covariance serves as a variance then it cant be non-negative
              # otherwise, it can be negative
              cov.k <- ifelse(cov.k < 0, 0, cov.k)
            }
            cor.k <- cov.k/(sqrt(var.k1)*sqrt(var.k2))
            cor.k <- ifelse(var.k1 == 0 | var.k2 == 0, ifelse(e == f, 1, 0), cor.k)
            cor.C[e, f] <- cor.k
            cor.C[f, e] <- cor.k
          }
        }
        mc.samples <- MASS::mvrnorm(n = mc.rep, rep(0, length = length(k)), cor.C, tol = 1)
        max.q <- vector(length = mc.rep)
        for(j in 1:mc.rep) {
          max.q[j] <- max(abs(mc.samples[j, ]))
        }
        # Should be 1-alpha, see Montiel Olea and Plagborg-Møller
        quant <- quantile(max.q, probs = 1-alpha)
      } else if(method == "theta-proj") {
        quant <- sqrt(qchisq(1-alpha, length(k)))
      } else if(method == "bonf") {
        quant <- qnorm(1-alpha/(2*length(k)))
      } else {
        stop("Invalid method selected")
      }
      for(j in seq_along(k)) {
        Lam <- Lam.vec[j]
        var.k <- ((k.c[j]*(1-k.c[j]))/(m*pi.0))*(1-2*Lam) +
          (Lam^2*(1-r[j])*r[j])/(m*pi.0^2)
        # Check to see if var.k is negative due to machine precision problem
        var.k <- ifelse(var.k < 0, 0, var.k)
        sd.k <- sqrt(var.k)
        lcl <- k.c[j] - quant*sd.k
        lcl <- ifelse(lcl < 0, 0, lcl)
        ucl <- k.c[j] + quant*sd.k
        ucl <- ifelse(ucl > k.ide[j], k.ide[j], ucl)
        CI.int[j, ] <- c(lcl, ucl)
      }
    } else if(metric == "prec") {
      if(method == "sup-t") {
        cor.C <- matrix(NA, ncol = length(k), nrow = length(k))
        for(f in seq_along(pi)) {
          for(e in 1:f) {
            Lam1 <- Lam.vec[e]
            var.pi1 <- (pi.c[e]*(1-pi.c[e]))/(m*r[e]) +
              (1-r[e])*(pi.c[e]-Lam1)^2/(m*r[e])
            var.pi1 <- ifelse(var.pi1 < 0, 0, var.pi1)
            Lam2 <- Lam.vec[f]
            var.pi2 <- (pi.c[f]*(1-pi.c[f]))/(m*r[f]) +
              (1-r[f])*(pi.c[f]-Lam2)^2/(m*r[f])
            var.pi2 <- ifelse(var.pi2 < 0, 0, var.pi2)
            cov.pi <- (((m*r[e]*r[f])^{-1})*(r[e]*pi.c[e]*(1 - pi.c[e]) +
                                               (pi.c[e]-Lam1)*(pi.c[e]-Lam2)*(r[e] - r[e]*r[f])))
            if(e == f) {
              # If the covariance serves as a variance then it cant be non-negative
              # otherwise, it can be negative
              cov.pi <- ifelse(cov.pi < 0, 0, cov.pi)
            }
            cor.pi <- cov.pi/(sqrt(var.pi1)*sqrt(var.pi2))
            cor.pi <- ifelse(var.pi1 == 0 | var.pi2 == 0, ifelse(e == f, 1, 0), cor.pi)
            cor.C[e, f] <- cor.pi
            cor.C[f, e] <- cor.pi
          }
        }
        mc.samples <- MASS::mvrnorm(n = mc.rep, rep(0, length = length(k)), cor.C, tol = 1)
        max.q <- vector(length = mc.rep)
        for(j in 1:mc.rep) {
          max.q[j] <- max(abs(mc.samples[j, ]))
        }
        quant <- quantile(max.q, probs = 1-alpha)
      } else if(method == "theta-proj") {
        quant <- sqrt(qchisq(1-alpha, length(pi)))
      } else if(method == "bonf") {
        quant <- qnorm(1-alpha/(2*length(k)))
      } else {
        stop("Invalid method selected")
      }
      for(j in seq_along(pi)) {
        Lam <- Lam.vec[j]
        var.pi <- (pi.c[j]*(1-pi.c[j]))/(m*r[j]) + (1-r[j])*(pi.c[j]-Lam)^2/(m*r[j])
        # Check to see if var.k is negative due to machine precision problem
        var.pi <- ifelse(var.pi < 0, 0, var.pi)
        sd.pi <- sqrt(var.pi)
        lcl <- pi.c[j] - quant*sd.pi
        lcl <- ifelse(lcl < 0, 0, lcl)
        ucl <- pi.c[j] + quant*sd.pi
        ucl <- ifelse(ucl > 1, 1, ucl)
        CI.int[j, ] <- c(lcl, ucl)
      }
    }
  }

  list(CI = CI.int, rec = k, prec = pi, h = h)

  }


#' Multiple variance Brownian motion estimation
#'
#' Computes rescaled branch lengths using multiple variance Brownian motion as described in Smaers et al. (2016)
#' @param data a vector of tip values for species; should be in the same order as tiplabels in the tree
#' @param tree an object of class "phylo".
#' @param sigma sig2 value from a Bayesian MCMC run using standard Brownian motion (e.g. ?anc.Bayes)
#' @return dataframe with rescaled branch lengths (rBL) for all branches in the tree
#' @references Smaers, Mongle & Kandler (2016) A multiple variance Brownian motion framework for estimating variable rates and inferring ancestral states. Biological Journal of the Linnean Society. 118 (1): 78-94.
#' @examples tree<-pbtree(n=50) # simulate tree ('pbtree' requires the 'phytools' package)
#' x<-fastBM(tree,sig2=2) # simulate values ('pbtree' requires the 'phytools' package)
#' BMsigma2<-ace(x,tree,method="REML")$sigma2[1] # get BM sigma2 ('ace' requires the 'ape' package)
#' mvBMresults<-mvBM(x,tree,BMsigma2) # calculate rescaled branch lengths using mvBM
#' tree_mvBM<-tree
#' tree_mvBM$edge.length<-mvBMresults$rBL # create new tree with rescaled branch lengths
#' plot(tree_mvBM) # plot mvBM tree
#' ace(x,tree_mvBM,method="REML") # get ancestral estimates using mvBM tree
#' # To obtain the mvBM branch specific rate estimate: calculate sig2 for the trait in question using the mvBM tree; multiply the mvBM sig2 with the mvBM rescaled branch length of the lineage of interest; divide this value by the phylogenetic branch length of the lineage of interest.
#' # MCMC posterior distributions can be obtained by using anc.Bayes for the above calculations. ('anc.Bayes' requires the 'phytools' package)
#' @examples for other examples see https://smaerslab.com/software/

#' @export
mvBM <- function (data, tree, sigma2)
{
    data_original <- data
    N = length(data)
    phy.matrix = data.frame(tree$edge, tree$edge.length, data[tree$edge[,
        2]])
    names(phy.matrix) = c("Anc", "Desc", "Length", "Value")
    nodes_extant = 1:N
    nodes_extinct = (N + 1):(N + (N - 1))
    extant_values <- data.frame(data, nodes_extant)
    Pk_values <- c()

    dist.tree <- dist.nodes(tree)

    for (j in nodes_extinct) {
        nominator = c()
        denominator = c()

        for (i in nodes_extant) {
            nominator = rbind(nominator, (extant_values[i, 1]/dist.tree[i,
                j]^2))
            denominator = rbind(denominator, (1/dist.tree[i,
                j]^2))
        }

        Pk = sum(nominator)/sum(denominator)
        Pk_values = rbind(Pk_values, Pk)
    }

    Pk <- c()
    Pk <- cbind(Pk, Pk_values)
    Pk <- cbind(Pk, nodes_extinct)
    colnames(Pk) <- c("value", "nodes")
    rownames(Pk) <- 1:length(nodes_extinct)
    Pk <- as.data.frame(Pk)
    nodes_extinct_reverse = sort(nodes_extinct, decreasing = TRUE)
    rBL <- c()
    rBL_edge <- c()
    for (i in nodes_extinct_reverse) {
        sister_branches <- which(phy.matrix$Anc == i)
        X1 <- phy.matrix$Value[sister_branches[1]]
        X2 <- phy.matrix$Value[sister_branches[2]]
        Y1 <- sqrt(phy.matrix$Length[sister_branches[1]])
        Y2 <- sqrt(phy.matrix$Length[sister_branches[2]])
        Pk_anc <- Pk$value[which(Pk$nodes == i)]
        Ax <- (Pk_anc + X1 + X2)/3
        T1 <- ((X1 - Ax)^2)/sigma2 + Y1^2
        T2 <- ((X2 - Ax)^2)/sigma2 + Y2^2
        phy.matrix$Value[which(phy.matrix$Desc == i)] <- Ax
        rBL <- c(rBL, T1)
        rBL <- c(rBL, T2)
        rBL_edge <- c(rBL_edge, sprintf("%05.f", sister_branches[1]))
        rBL_edge <- c(rBL_edge, sprintf("%05.f", sister_branches[2]))
        names(rBL) <- rBL_edge
    }
    Output <- data.frame(tree$edge, tree$edge.length, rBL[sort(names(rBL))])
    names(Output) <- c("node_anc", "node_desc", "BL", "rBL")
    return(Output)
}


model_phylsowingdatecorrection <- function (sowingDay = 1,
         latitude = 0.0,
         sDsa_sh = 1.0,
         rp = 0.0,
         sDws = 1,
         sDsa_nh = 1.0,
         p = 120.0){
    #'- Name: PhylSowingDateCorrection -Version: 1.0, -Time step: 1
    #'- Description:
    #'            * Title: PhylSowingDateCorrection Model
    #'            * Author: Loic Manceau
    #'            * Reference: Modeling development phase in the
    #'                Wheat Simulation Model SiriusQuality.
    #'                See documentation at http://www1.clermont.inra.fr/siriusquality/?page_id=427
    #'            * Institution: INRA Montpellier
    #'            * Abstract: Correction of the Phyllochron Varietal parameter according to sowing date
    #'- inputs:
    #'            * name: sowingDay
    #'                          ** description : Day of Year at sowing
    #'                          ** parametercategory : species
    #'                          ** datatype : INT
    #'                          ** min : 1
    #'                          ** max : 365
    #'                          ** default : 1
    #'                          ** unit : d
    #'                          ** uri : some url
    #'                          ** inputtype : parameter
    #'            * name: latitude
    #'                          ** description : Latitude
    #'                          ** parametercategory : soil
    #'                          ** datatype : DOUBLE
    #'                          ** min : -90
    #'                          ** max : 90
    #'                          ** default : 0.0
    #'                          ** unit : °
    #'                          ** uri : some url
    #'                          ** inputtype : parameter
    #'            * name: sDsa_sh
    #'                          ** description : Sowing date at which Phyllochrone is maximum in southern hemispher
    #'                          ** parametercategory : species
    #'                          ** inputtype : parameter
    #'                          ** datatype : DOUBLE
    #'                          ** min : 1
    #'                          ** max : 365
    #'                          ** default : 1.0
    #'                          ** unit : d
    #'                          ** uri : some url
    #'            * name: rp
    #'                          ** description : Rate of change of Phyllochrone with sowing date
    #'                          ** parametercategory : species
    #'                          ** inputtype : parameter
    #'                          ** datatype : DOUBLE
    #'                          ** min : 0
    #'                          ** max : 365
    #'                          ** default : 0
    #'                          ** unit : d-1
    #'                          ** uri : some url
    #'            * name: sDws
    #'                          ** description : Sowing date at which Phyllochrone is minimum
    #'                          ** parametercategory : species
    #'                          ** datatype : INT
    #'                          ** default : 1
    #'                          ** min : 1
    #'                          ** max : 365
    #'                          ** unit : d
    #'                          ** uri : some url
    #'                          ** inputtype : parameter
    #'            * name: sDsa_nh
    #'                          ** description : Sowing date at which Phyllochrone is maximum in northern hemispher
    #'                          ** parametercategory : species
    #'                          ** datatype : DOUBLE
    #'                          ** default : 1.0
    #'                          ** min : 1
    #'                          ** max : 365
    #'                          ** unit : d
    #'                          ** uri : some url
    #'                          ** inputtype : parameter
    #'            * name: p
    #'                          ** description : Phyllochron (Varietal parameter)
    #'                          ** parametercategory : species
    #'                          ** datatype : DOUBLE
    #'                          ** default : 120
    #'                          ** min : 0
    #'                          ** max : 1000
    #'                          ** unit : °C d leaf-1
    #'                          ** uri : some url
    #'                          ** inputtype : parameter
    #'- outputs:
    #'            * name: fixPhyll
    #'                          ** description :  Phyllochron Varietal parameter
    #'                          ** variablecategory : auxiliary
    #'                          ** datatype : DOUBLE
    #'                          ** min : 0
    #'                          ** max : 1000
    #'                          ** unit : °C d leaf-1
    if (latitude < 0.0)
    {
        if (sowingDay > as.integer(sDsa_sh))
        {
            fixPhyll <- p * (1 - (rp * min((sowingDay - sDsa_sh), sDws)))
        }
        else
        {
            fixPhyll <- p
        }
    }
    else
    {
        if (sowingDay < as.integer(sDsa_nh))
        {
            fixPhyll <- p * (1 - (rp * min(sowingDay, sDws)))
        }
        else
        {
            fixPhyll <- p
        }
    }
    return (list('fixPhyll' = fixPhyll))
}


#' @export
#'
#' @title checkIsAutocorrelated - Check whether MCMC chains are autocorrelated.
#'
#' @description Run autocorrelation on every parameter in an mcmc.list and make
#' a decision whether they are significantly auto correlated.  Ideally, chains
#' are not autocorrelated when converged.
#'
#' @param obj An mcmc.list object from the \code{rjags} package.  These are output
#' by the \code{coda.samples} function.
#'
#' @param criterion scalar specifying the level of acceptable autocorrelation at
#' the assessed \code{lag}.  If all average lag2 autocorrelations are less than this number, the
#' whole mcmc object is deems 'random' or not autocorrelated.
#'
#' @param lag The lag at which to assess autocorrelation.  This should generally be a
#' low number like 1, 2, 5, or 10.
#'
#' @param quiet Whether to print any output on screen.
#'
#' @details This routine calculates auto correlation separately at the specified lag
#' for every chain in the mcmc.list, then averages autocorrelations across
#' chains.
#'
#' @return A list with the following components
#' \itemize{
#'   \item autoCorrelated: If the average autocorrelations at the specified
#'   lag are all less than
#'   \code{criterion}, the chain is declared un-autocorrelated and is set FALSE.
#'   Otherwise, this component of the returns is TRUE.
#'   \item autoCorrs: the computed autocorrelations at the specified lag.
#'   This value is the average across chains in \code{obj}.
#'  }
#'
#' @author Trent McDonald
#'
#'
#' @seealso \code{\link{checkIsConverged}}
#'
#' @examples
#' \dontrun{
#' jags = jags.model(file="someJAGSFile.txt",
#'    data=someJAGS.data, inits=someJAGS.inits,
#'    n.chains=3,n.adapt=100)
#' update(jags, n.iter=1000)
#' out = coda.samples(jags,
#'    variable.names=c("someParameter"),
#'    n.iter=1000,
#'    thin=2)
#'
#' auto <- chechIsAutocorrelated(out)
#' }

checkIsAutocorrelated <- function(obj, criterion=0.4, lag=1, quiet=FALSE){

  if(!quiet) cat(paste0("\n", paste(rep("-",4),collapse="")," Checking autocorrelation:\n"))
  vn <- varnames(obj)
  ncovars <- nvar(obj)
  nchains <- nchain(obj)
  acflag2 <- rep(NA,ncovars)
  names(acflag2) <- vn
  for( i in 1:ncovars ){
    tmp <- rep(NA,nchains)
    for(j in 1:nchains){
      tmp.2 <- acf(as.matrix(obj[[j]][,i]), lag.max = lag, plot=FALSE)
      tmp[j] <- tmp.2$acf[tmp.2$lag ==lag,,]
    }
    acflag2[i] <- mean(tmp)
    if(!quiet) cat(paste( vn[i], ": mean(acf[",lag, "]) =", round(acflag2[i],5), "\n") )
  }
  if(any(acflag2>criterion, na.rm=TRUE)){
    if(!quiet) cat(paste0("\nSome autocorrelations exceed ",criterion, ".\nInspect acfplot(mcmc.list) and increase thinning.\n"))
    ans <- TRUE
  } else {
    ans <- FALSE
  }

  list(autoCorrelated = ans, autoCorrs=acflag2)

}


##  6장 소스코드

# CSV 파일 읽어들이기
ad.data <- read.csv("ad_result.csv", header = T, stringsAsFactors = F)
ad.data

# TV 광고의 광고비용과 신규 유저수의 산점도를 그리기
install.packages("ggplot2")
install.packages("scales")
library(ggplot2)
library(scales)

ggplot(ad.data, aes(x = tvcm, y = install)) + geom_point() +
  xlab("TV 광고비") + ylab("신규 유저수") +
  scale_x_continuous(label = comma) +
  scale_y_continuous(label = comma) + geom_smooth(method="lm")

# 잡지매체의 광고비용과 신규 유저수의 산점도를 그리기
ggplot(ad.data, aes(x = magazine, y = install)) + geom_point() +
  xlab("잡지 광고비") + ylab("신규 유저수") +
  scale_x_continuous(label = comma) +
  scale_y_continuous(label = comma) + geom_smooth(method="lm")

# 회귀분석 실행
fit <- lm(install ~ ., data = ad.data[, c("install", "tvcm", "magazine")])
fit

# 회귀분석 결과를 해석하기
summary(fit)
options(scipen=1000)


#'@title calcKristCa
#'
#'@description Calculate incremental hull (roughness) resistance coefficient
#'(\code{Ca}) (dimensionless) using the Kristensen method.
#'
#'@param shipType Ship type (vector of strings, see \code{\link{calcShipType}}).
#'Must align with \code{tankerBulkCarrierGCargoShipTypes} and
#' \code{containerShipTypes} groupings
#'@param actualDisplacement Actual loaded displacement (vector of numericals,
#' m^3) (see \code{\link{calcActualDisp}})
#'@param tankerBulkCarrierGCargoShipTypes Ship types specified in input
#'\code{shipTypes} to be modeled as tankers, bulk carriers and general cargo
#'vessels (vector of strings)
#'@param containerShipTypes Ship types specified in input \code{shipTypes} to be
#'modeled as container ships (vector of strings)
#'
#'@details
#'Models the effect of realistic hull roughness on resistance, which is not
#'captured in the frictional and residual resistance coefficients from tank
#'towing operations.
#'
#' This method this requires ship types to be grouped. Use the
#' \code{tankerBulkCarrierGCargoShipTypes}, \code{containerShipTypes} grouping
#' parameters to provide these ship type groupings. Any ship types not included
#' in these groupings will be considered as miscellaneous
#'
#'@return \code{Ca} (vector of numericals, dimensionless)
#'
#'@references
#'Kristensen, H. O. and Lutzen, M. 2013. "Prediction of Resistance and Propulsion
#'Power of Ships."
#'
#'\href{https://gitlab.gbar.dtu.dk/oceanwave3d/Ship-Desmo}{Kristensen, H. O.
#'"Ship-Desmo-Tool." https://gitlab.gbar.dtu.dk/oceanwave3d/Ship-Desmo}
#'
#'@seealso \code{\link{calcActualDisp}}
#'
#'@family Kristensen Calculations
#'@family Resistance Calculations
#'
#'@examples
#'calcKristCa(c("bulk.carrier","container.ship"),c(73663.27,216726.45))
#'
#'@export

calcKristCa<-function(shipType,actualDisplacement,
                      tankerBulkCarrierGCargoShipTypes=c("tanker","general.cargo","chemical.tanker","liquified.gas.tanker","oil.tanker","other.tanker","bulk.carrier"),
                      containerShipTypes=c("container.ship")){

Ca<- ifelse(shipType%in%containerShipTypes,
            (0.5*log10(actualDisplacement)-0.1*(log10(actualDisplacement))^2)/1000,
            ifelse(shipType%in%tankerBulkCarrierGCargoShipTypes,
  (pmax(-0.1, 0.5*log10(actualDisplacement)-0.1*(log10(actualDisplacement))^2))/1000,
  NA
)
)
return(Ca)
}


# d: dataset, m: mean, sd: width, a: aplitude
vOrigDeriv = Inf

aApproxDeriv = 0
mApproxDeriv = 0
sdApproxDeriv = 0

dataDeriv <- convolve(data, c(-1,1), type= 'filter')

for (sd in sdTab) {
  for(m in mTab) {
    dnFast <- dnorm(1:length(dataDeriv), mean = m, sd = sd)

    n <- (log2Population - 1):1
    aTabPow <- 2 ** n
    aApproxCur = 2 ** log2Population
    vAtAApproxCur = Inf
    for (a in aTabPow) {
      v <- var(dataDeriv - (dnFast * aApproxCur))
      vAp <- var(dataDeriv - (dnFast * (aApproxCur + a)))
      vAm <- var(dataDeriv - (dnFast * (aApproxCur - a)))

      if (v > vAp && vAm > vAp) {
        aApproxCur = aApproxCur + a
      } else if (v > vAm && vAp > vAm) {
        aApproxCur = aApproxCur - a
      }
      if (v < vAtAApproxCur) {
        aApproxCurMin = aApproxCur
        vAtAApproxCur = v
      }
    }
    if (vAtAApproxCur < vOrigDeriv) {
      sdApproxDeriv = sd
      mApproxDeriv = m
      aApproxDeriv = aApproxCurMin
      vOrigDeriv = vAtAApproxCur
    }
  }
}
print('sdApproxDeriv: ')
print(sdApproxDeriv)
print('mApproxDeriv: ')
print(mApproxDeriv)
print('aApproxDeriv: ')
print(aApproxDeriv)
print('vOrigDeriv: ')
print(vOrigDeriv)


aux_step03_Weight_of_Norm_matrix <-function(inMatrix, inWeight){

  outMat <- inMatrix
  ncols<-dim(inMatrix)[2]
  nrows<-dim(inMatrix)[1]

  #iMcalc2 <- matrix(inMatrix[,2:ncols], nrow=nrows, ncol=ncols-1, byrow=TRUE)
  iMcalc <- matrix(inMatrix[,2:ncols], nrow=nrows, ncol=ncols-1, byrow=FALSE)

  nC <- dim(iMcalc)[2] # Convert weigths into a matrix in order to perform Mult
  aMmul <- matrix(rep(inWeight, times=nrows), ncol=nC, byrow=TRUE)

  auxCalc <- iMcalc * aMmul

  outMat[,2:ncols]<-auxCalc

  return(outMat)
}

#'
#' @param Matrices with Benefits and Costs,
#' @param Vectors with Benefits and Costs weights,
#' @return Score of Paths
#' @author Bruno Sousa
#' @note v1.0 no arguments validation
#' @title METH_runTOPSIS
#' @name METH_runTOPSIS
METH_runTOPSIS <- function(iMBen, iMCost, iVecBen,iVecCost ){
  # Important Global Variables
  MIN_COST_TOPSIS_TENDENCY <- 0.0001  # To avoid divisions by zero
  BETA_TOPSIS_TENDENCY_WEIGHTS <- 0.5
  MINSUM_TOPSIS <- 1e-99 # To avoid divisions by zero in normalization

  #source("libNormalization.R")

  mBen_Criteria <- iMBen
  mCost_Criteria <- iMCost

  vBen_weight <- iVecBen
  vCost_weight <- iVecCost

  # Euclidean Distance
  fAux_Euclidean_Dist<- function(i1, i2){
    adif <- (i1 - i2)^2

    return(adif)
  }

  #
  # Convert weight costs into Benefit costs
  #
  aux_step01_Tendency_Cnv <- function(inCosts){
    nCol <- dim(inCosts)[2]
    aux <- matrix(inCosts[,2:nCol], ncol=nCol-1)

    aux[which(aux==0) ] <- MIN_COST_TOPSIS_TENDENCY
    aux <- 1/aux

    return (aux)
  }

  #
  # Append Benefits and Cost weigths
  #
  aux_step01_Tendend_Weigths <- function(inWeBen, inWeCost){

    ret_Ben  <-    BETA_TOPSIS_TENDENCY_WEIGHTS * inWeBen
    ret_Cost <- (1 - BETA_TOPSIS_TENDENCY_WEIGHTS) * inWeCost

    ret_ <- c(ret_Ben, ret_Cost)

    return(ret_)
  }

  #
  # Normalize matrix
  # Using Vector normalization instead of min-max
  #
  aux_step02_Normalization <- function(inMatrix){

    #internal function to help in normalization
    fSum <- function(i){
      sumrow <- MINSUM_TOPSIS
      aSu <- sum(i^2) + sumrow
      return(aSu)
    }

    outMat <- inMatrix
    ncols<-dim(inMatrix)[2]
    nrows<-dim(inMatrix)[1]

    iMcalc <- matrix(inMatrix[,2:ncols], nrow=nrows, ncol=ncols-1, byrow=TRUE)

    #Apply Sum by col
    auxSum <- apply(iMcalc, 2, FUN=fSum)

    nC <- dim(iMcalc)[2]
    aMSum <- matrix(rep(auxSum, times=nrows), ncol=nC, byrow=TRUE)
    auxCalc <- iMcalc / sqrt(aMSum)

    outMat[,2:ncols]<-auxCalc
    return(outMat)

  }

  #
  # Determine PIS according to DiA Method
  #
  aux_step04_PIS <-function(inM){
    ncolu <- dim(inM)[2]
    nrows <- dim(inM)[1]
    iMcalc <- matrix(inM[,2:ncolu], nrow=nrows, ncol=ncolu-1, byrow=TRUE)
    auxRet <- apply(iMcalc, 2, max)
    return (auxRet)
  }

  aux_step04_NIS<-function(inM){
    ncolu <- dim(inM)[2]
    nrows <- dim(inM)[1]
    iMcalc <- matrix(inM[,2:ncolu], nrow=nrows, ncol=ncolu-1, byrow=TRUE)
    auxRet <- apply(iMcalc, 2, min)
    return (auxRet)
  }

  #
  # Determine the distance to the ideal points (PIS and NIS)
  #
  aux_step05_distance_to_ideal <- function(inApos,  inMatrix){
    #Be careful with the idx of Ideal....
    ncols<- dim(inMatrix)[2]
    nrows<- dim(inMatrix)[1]

    auxMBenef <- matrix(ncol=2,nrow=nrows)
    auxMBenef[,1] <- inMatrix[,1]

    mCalc <- matrix(inMatrix[,2:ncols], nrow=nrows, ncol=ncols-1, byrow=FALSE)

    for (nR in seq(from=1, to=nrows)){
      aSumBen <- 0
      for (nC in seq(from=1, to=ncol(mCalc))){
        aSumBen <- aSumBen + fAux_Euclidean_Dist(mCalc[nR, nC] , inApos[nC])
      }
      auxMBenef[nR,2] <- aSumBen
    }

    ret_ <- auxMBenef
    return (ret_)
  }

  #
  # Determine R score
  #
  aux_step06_Ranking<-function(inSepPos, inSepNeg){

    nRow <- dim(inSepPos)[1]
    nCol <- dim(inSepPos)[2]
    auxCi <- matrix(ncol=2, nrow=nRow)
    auxCi[,1] <- inSepPos[,1]
    mCalc <- matrix(c(inSepPos[,2], inSepNeg[,2]), ncol=2, nrow=nRow, byrow=FALSE)

    fDist <- function(i){ # Apply a function on pairwise columns
      ipos <- mCalc[i,1]
      ineg <- mCalc[i,2]
      auxCj <- ineg / (ipos + ineg)
      return (auxCj)
    }

    # dist
    aDist <- sapply(1:nrow(mCalc), fDist)
    auxCi[,2] <- aDist
    return(auxCi)
  }

  ncolMB <- dim(mBen_Criteria)[2]
  TPSTOPsisBenefits <- as.matrix(mBen_Criteria )

  ncolMC <- dim(mCost_Criteria)[2]
  TPSTOPsisCosts <- as.matrix(mCost_Criteria )

  TPSWeiBenTOP <- vBen_weight
  TPSWeiCostTOP <- vCost_weight

  #
  # Step 01
  #
  TPSTOPsisCostsTended <- aux_step01_Tendency_Cnv(TPSTOPsisCosts)
  TPSWeiBenTOPall <- aux_step01_Tendend_Weigths(TPSWeiBenTOP, TPSWeiCostTOP)
  TPSTOPsisBenefits <- cbind(TPSTOPsisBenefits, TPSTOPsisCostsTended)
  TPSWeiBenTOP <- TPSWeiBenTOPall
  stopifnot(TPSTOPsisBenefits != NULL)

  #
  # Step 02
  #
  TPSTOPsisBenefitsTOP <- aux_step02_Normalization(TPSTOPsisBenefits)

  #
  # Step 03
  #
  TPSTOPsisBenefitsTOP <- aux_step03_Weight_of_Norm_matrix(TPSTOPsisBenefitsTOP, TPSWeiBenTOP)

  # Step 04 - Ideal Solutions
  TPSApos_Dia <- aux_step04_PIS(TPSTOPsisBenefitsTOP)
  TPSAneg_Dia <- aux_step04_NIS(TPSTOPsisBenefitsTOP)

  # Step 05 - Distance to Ideal
  TPSDposBen_Dia <- aux_step05_distance_to_ideal(TPSApos_Dia, TPSTOPsisBenefitsTOP )
  TPSDnegBen_Dia <- aux_step05_distance_to_ideal(TPSAneg_Dia, TPSTOPsisBenefitsTOP)

  #Step 06
  #RdistBen_Dia  <- aux_step06_Rscore(DposBen_Dia, DnegBen_Dia)
  TPStopsisRanking <- aux_step06_Ranking(TPSDposBen_Dia, TPSDnegBen_Dia)
  #TPStopsisRanking

  #print(TPStopsisRanking)
  return(TPStopsisRanking[order(TPStopsisRanking[,2]),])
}

#'
#' @param Matrices with Benefits and Costs,
#' @param Vectors with Benefits and Costs weights,
#' @return Score of Paths
#' @author Bruno Sousa
#' @note v1.0 no arguments validation
#' @title METH_runDiA
#' @name METH_runDiA
METH_runDiA <- function(iMBen, iMCost, iVecBen,iVecCost ){
  # Important Global Variables
  MIN_COST_TOPSIS_TENDENCY <- 0.0001  # To avoid divisions by zero
  BETA_TOPSIS_TENDENCY_WEIGHTS <- 0.5
  MINSUM_TOPSIS <- 1e-99 # To avoid divisions by zero in normalization

  #source("libNormalization.R")

  mBen_Criteria <- iMBen
  mCost_Criteria <- iMCost

  vBen_weight <- iVecBen
  vCost_weight <- iVecCost

  #
  # Convert weight costs into Benefit costs
  #
  aux_step01_Tendency_Cnv <- function(inCosts){
    nCol <- dim(inCosts)[2]
    aux <- matrix(inCosts[,2:nCol], ncol=nCol-1)

    aux[which(aux==0) ] <- MIN_COST_TOPSIS_TENDENCY
    aux <- 1/aux

    return (aux)
  }

  #
  # Append Benefits and Cost weigths
  #
  aux_step01_Tendend_Weigths <- function(inWeBen, inWeCost){

    ret_Ben  <-    BETA_TOPSIS_TENDENCY_WEIGHTS * inWeBen
    ret_Cost <- (1 - BETA_TOPSIS_TENDENCY_WEIGHTS) * inWeCost

    ret_ <- c(ret_Ben, ret_Cost)

    return(ret_)
  }

  #
  # Normalize matrix
  # Using Vector normalization instead of min-max
  #
  aux_step02_Normalization <- function(inMatrix){

    #internal function to help in normalization
    fSum <- function(i){
      sumrow <- MINSUM_TOPSIS
      aSu <- sum(i^2) + sumrow
      return(aSu)
    }

    outMat <- inMatrix
    ncols<-dim(inMatrix)[2]
    nrows<-dim(inMatrix)[1]

    iMcalc <- matrix(inMatrix[,2:ncols], nrow=nrows, ncol=ncols-1, byrow=TRUE)

    #Apply Sum by col
    auxSum <- apply(iMcalc, 2, FUN=fSum)

    nC <- dim(iMcalc)[2]
    aMSum <- matrix(rep(auxSum, times=nrows), ncol=nC, byrow=TRUE)
    auxCalc <- iMcalc / sqrt(aMSum)

    outMat[,2:ncols]<-auxCalc
    return(outMat)

  }

  #
  # Determine PIS according to DiA Method
  #
  aux_step04_PIS <-function(inM){
    ncolu <- dim(inM)[2]
    nrows <- dim(inM)[1]
    iMcalc <- matrix(inM[,2:ncolu], nrow=nrows, ncol=ncolu-1, byrow=TRUE)
    auxRet <- apply(iMcalc, 2, max)
    return (auxRet)
  }

  aux_step04_NIS<-function(inM){
    ncolu <- dim(inM)[2]
    nrows <- dim(inM)[1]
    iMcalc <- matrix(inM[,2:ncolu], nrow=nrows, ncol=ncolu-1, byrow=TRUE)
    auxRet <- apply(iMcalc, 2, min)
    return (auxRet)
  }

  #
  # Determine the distance to the ideal points (PIS and NIS)
  #
  aux_step05_distance_to_ideal <- function(inApos,  inMatrix){
    #Be careful with the idx of Ideal....
    ncols<- dim(inMatrix)[2]
    nrows<- dim(inMatrix)[1]

    auxMBenef <- matrix(ncol=2,nrow=nrows)
    auxMBenef[,1] <- inMatrix[,1]

    mCalc <- matrix(inMatrix[,2:ncols], nrow=nrows, ncol=ncols-1, byrow=FALSE)
    #internal fx
    fAux_Positive_Dif <- function(i1, i2){
      adif <- (i1 - i2)
      if (adif < 0) {
        adif <- adif * (-1)
      }
      return(adif)
    }

    for (nR in seq(from=1, to=nrows)){
      aSumBen <- 0
      for (nC in seq(from=1, to=ncol(mCalc))){
        aSumBen <- aSumBen + fAux_Positive_Dif(mCalc[nR, nC] , inApos[nC])
      }
      auxMBenef[nR,2] <- aSumBen
    }

    ret_ <- auxMBenef
    return (ret_)
  }

  #
  # Determine R score
  #
  aux_step06_Rscore<-function(inSepPos, inSepNeg){

    nRow <- dim(inSepPos)[1]
    nCol <- dim(inSepPos)[2]
    auxCi <- matrix(ncol=2, nrow=nRow)
    auxCi[,1] <- inSepPos[,1]
    mCalc <- matrix(c(inSepPos[,2], inSepNeg[,2]), ncol=2, nrow=nRow, byrow=FALSE)

    # get ideal point
    PIA_pos <- min(inSepPos[,2])
    PIA_neg <- max(inSepNeg[,2])

    fDist <- function(i){ # Apply a function on pairwise columns
      ipos <- mCalc[i,1]
      ineg <- mCalc[i,2]
      auxSqrt <- sqrt( (ipos - PIA_pos)^2 + (ineg - PIA_neg)^2 )
      return (auxSqrt)
    }

    # dist
    aDist <- sapply(1:nrow(mCalc), fDist)
    auxCi[,2] <- aDist
    return(auxCi)
  }

  ncolMB <- dim(mBen_Criteria)[2]
  DiATOPsisBenefits <- as.matrix(mBen_Criteria )

  ncolMC <- dim(mCost_Criteria)[2]
  DiATOPsisCosts <- as.matrix(mCost_Criteria )

  DiAWeiBenTOP <- vBen_weight
  DiAWeiCostTOP <- vCost_weight

  #
  # Step 01
  #
  DiATOPsisCostsTended <- aux_step01_Tendency_Cnv(DiATOPsisCosts)
  DiAWeiBenTOPall <- aux_step01_Tendend_Weigths(DiAWeiBenTOP, DiAWeiCostTOP)
  DiATOPsisBenefits <- cbind(DiATOPsisBenefits, DiATOPsisCostsTended)
  DiAWeiBenTOP <- DiAWeiBenTOPall
  stopifnot(DiATOPsisBenefits != NULL)

  #
  # Step 02
  #
  DiATOPsisBenefitsTOP <- aux_step02_Normalization(DiATOPsisBenefits)
  DiATOPsisBenefitsTOPNorm <- DiATOPsisBenefitsTOP

  #
  # Step 03
  #
  DiATOPsisBenefitsTOP <- aux_step03_Weight_of_Norm_matrix(DiATOPsisBenefitsTOP, DiAWeiBenTOP)

  # Step 04
  DiAApos_Dia <- aux_step04_PIS(DiATOPsisBenefitsTOP)
  DiAAneg_Dia <- aux_step04_NIS(DiATOPsisBenefitsTOP)

  # Step 05
  DiADposBen_Dia <- aux_step05_distance_to_ideal(DiAApos_Dia, DiATOPsisBenefitsTOP )
  DiADnegBen_Dia <- aux_step05_distance_to_ideal(DiAAneg_Dia, DiATOPsisBenefitsTOP)

  #Step 06
  DiARdistBen_Dia  <- aux_step06_Rscore(DiADposBen_Dia, DiADnegBen_Dia)
  #DiARdistBen_Dia

  #print(DiARdistBen_Dia)
  return(DiARdistBen_Dia[order(DiARdistBen_Dia[,2]),])

}

#'
#' @param Matrices with Benefits and Costs,
#' @param Vectors with Benefits and Costs weights,
#' @return Score of Paths
#' @author Bruno Sousa
#' @note v1.0 no arguments validation
#' @title METH_runNMMD
#' @name METH_runNMMD
METH_runNMMD <- function(iMBen, iMCost, iVecBen,iVecCost ){
  require("corpcor")
  require("HDMD")

  # Important Global Variables
  MIN_COST_TOPSIS_TENDENCY <- 0.0001  # To avoid divisions by zero
  BETA_TOPSIS_TENDENCY_WEIGHTS <- 0.5
  MINSUM_TOPSIS <- 1e-99 # To avoid divisions by zero in normalization

  #source("libNormalization.R")

  mBen_Criteria <- iMBen
  mCost_Criteria <- iMCost

  vBen_weight <- iVecBen
  vCost_weight <- iVecCost

  # Function to check if Mahalanobis can be processed
  # This is to avoid errors on singular matrices.
  # TODO: need to understand why this can happen
  #
  # TODO2: Need a positive definite matrix... Probably due to the fact of having few data
  #        Some useful urls:
  #        http://www.r-bloggers.com/dealing-with-non-positive-definite-matrices-in-r/
  #   http://www2.gsu.edu/~mkteer/npdmatri.html
  fAux_Do_Mahalanobis <- function(iM){
    ret <- TRUE
    #auxCov  <- cov(iM)
    auxCov2 <- cov.shrink(iM,verbose=FALSE)

    #if (det(auxCov)==0){ ret <- FALSE }
    if (!is.positive.definite(auxCov2)){
      ret <- FALSE
    }
    return (ret)
  }

  #Mahalanobis Distance
  fAux_Mahalanobis_Dist <- function(iM, me=NULL){

    if (length(me)==0){
      auxMean <- apply(iM, 2, mean)
    }else{
      auxMean <- me
    }
    #auxCov  <- cov(iM)
    auxCov  <- cov.shrink(iM, verbose=FALSE)

    # To avoid errors
    if (fAux_Do_Mahalanobis(iM)){
      #print(paste("doing Maha for"  ))
      #print(iM)
      #print(auxCov)
      #print(det(auxCov))
      auxDistMaha <- mahalanobis(iM, auxMean, auxCov)
      return (auxDistMaha)
    }else{
      return (-1)
    }
  }

  myPairwise.mahalanobis <- function (x, grouping = NULL, cov = NULL, inverted = FALSE, digits = 5, ...)
  {
    x <- if (is.vector(x))
      matrix(x, ncol = length(x))
    else as.matrix(x)
    if (!is.matrix(x))
      stop("x could not be forced into a matrix")
    if (length(grouping) == 0) {
      grouping = t(x[1])
      x = x[2:dim(x)[2]]
      cat("assigning grouping\n")
      print(grouping)
    }
    n <- nrow(x)
    p <- ncol(x)
    if (n != length(grouping)) {
      cat(paste("n: ", n, "and groups: ", length(grouping),
                "\n"))
      stop("nrow(x) and length(grouping) are different")
    }
    g <- as.factor(grouping)
    g
    lev <- lev1 <- levels(g)
    counts <- as.vector(table(g))
    if (any(counts == 0)) {
      empty <- lev[counts == 0]
      warning(sprintf(ngettext(length(empty), "group %s is empty",
                               "groups %s are empty"), paste(empty, collapse = " ")),
              domain = NA)
      lev1 <- lev[counts > 0]
      g <- factor(g, levels = lev1)
      counts <- as.vector(table(g))
    }
    ng = length(lev1)
    group.means <- tapply(x, list(rep(g, p), col(x)), mean)
    if (missing(cov)) {
      inverted = FALSE
      cov = cor(x)
    }
    else {
      if (dim(cov)[1] != p && dim(cov)[2] != p)
        stop("cov matrix not of dim = (p,p)\n")
    }
    Distance = matrix(nrow = ng, ncol = ng)
    dimnames(Distance) = list(names(group.means), names(group.means))
    Means = round(group.means, digits)
    Cov = round(cov, digits)
    Distance = round(Distance, digits)
    for (i in 1:ng) {
      Distance[i, ] = mahalanobis(group.means, group.means[i,], cov, inverted)
    }
    result <- list(means = group.means, cov = cov, distance = Distance)
    result
  }

  #Mahalanobis Distance
  fAux_Pairwise.Mahalanobis_Dist <- function(iM, me=NULL){

    if (length(me)==0){
      auxMean <- apply(iM, 2, mean)
    }else{
      auxMean <- me
    }
    #auxCov  <- cov(iM)
    auxCov  <- cov.shrink(iM, verbose=FALSE)

    # To avoid errors
    if (fAux_Do_Mahalanobis(iM)){
      #print(paste("doing Maha for"  ))
      #print(iM)
      #print(det(auxCov))
      #print(ncol(iM))
      #auxDistMaha <- pairwise.mahalanobis(iM, grouping=t(iM[,1]), digits=5, center=auxMean, cov=auxCov)
      auxDistMaha <- myPairwise.mahalanobis(iM, grouping=t(iM[,1]), digits=5, center=auxMean, cov=auxCov)
      return (auxDistMaha)
    }else{
      return (-1)
    }
  }

  #
  # Convert weight costs into Benefit costs
  #
  aux_step01_Tendency_Cnv <- function(inCosts){
    nCol <- dim(inCosts)[2]
    aux <- matrix(inCosts[,2:nCol], ncol=nCol-1)

    aux[which(aux==0) ] <- MIN_COST_TOPSIS_TENDENCY
    aux <- 1/aux

    return (aux)
  }

  #
  # Append Benefits and Cost weigths
  #
  aux_step01_Tendend_Weigths <- function(inWeBen, inWeCost){

    ret_Ben  <-    BETA_TOPSIS_TENDENCY_WEIGHTS * inWeBen
    ret_Cost <- (1 - BETA_TOPSIS_TENDENCY_WEIGHTS) * inWeCost

    ret_ <- c(ret_Ben, ret_Cost)

    return(ret_)
  }

  #
  # Normalize matrix
  # Using Vector normalization instead of min-max
  #
  aux_step02_Normalization <- function(inMatrix){

    #internal function to help in normalization
    fSum <- function(i){
      sumrow <- MINSUM_TOPSIS
      aSu <- sum(i^2) + sumrow
      return(aSu)
    }

    outMat <- inMatrix
    ncols<-dim(inMatrix)[2]
    nrows<-dim(inMatrix)[1]

    iMcalc <- matrix(inMatrix[,2:ncols], nrow=nrows, ncol=ncols-1, byrow=TRUE)

    #Apply Sum by col
    auxSum <- apply(iMcalc, 2, FUN=fSum)

    nC <- dim(iMcalc)[2]
    aMSum <- matrix(rep(auxSum, times=nrows), ncol=nC, byrow=TRUE)
    auxCalc <- iMcalc / sqrt(aMSum)

    outMat[,2:ncols]<-auxCalc
    return(outMat)

  }

  #
  # Weight the normalized matrix, by multiplying each value by its weight
  #
  aux_step03_Weight_of_Norm_matrix <-function(inMatrix, inWeight){

    outMat <- inMatrix
    ncols<-dim(inMatrix)[2]
    nrows<-dim(inMatrix)[1]

    iMcalc <- matrix(inMatrix[,2:ncols], nrow=nrows, ncol=ncols-1, byrow=TRUE)

    nC <- dim(iMcalc)[2] # Convert weigths into a matrix in order to perform Mult
    aMmul <- matrix(rep(inWeight, times=nrows), ncol=nC, byrow=TRUE)

    auxCalc <- iMcalc * aMmul

    outMat[,2:ncols]<-auxCalc

    return(outMat)
  }

  #
  # Determine PIS according to DiA Method
  #
  aux_step04_PIS <-function(inM){
    ncolu <- dim(inM)[2]
    nrows <- dim(inM)[1]
    iMcalc <- matrix(inM[,2:ncolu], nrow=nrows, ncol=ncolu-1, byrow=TRUE)
    auxRet <- apply(iMcalc, 2, max)
    return (auxRet)
  }

  aux_step04_NIS<-function(inM){
    ncolu <- dim(inM)[2]
    nrows <- dim(inM)[1]
    iMcalc <- matrix(inM[,2:ncolu], nrow=nrows, ncol=ncolu-1, byrow=TRUE)
    auxRet <- apply(iMcalc, 2, min)
    return (auxRet)
  }

  #
  # Determine the distance to the ideal points (PIS and NIS)
  #
  aux_step05_distance_to_idealNMMD <- function(inMatrix){
    #Be careful with the idx of Ideal....
    ncols<- dim(inMatrix)[2]
    nrows<- dim(inMatrix)[1]

    #auxMBenef <- matrix(ncol=2,nrow=nrows)
    #auxMBenef[,1] <- inMatrix[,1]

    mCalc <- matrix(inMatrix[,2:ncols], nrow=nrows, ncol=ncols-1, byrow=FALSE)

    auxMBenef <- fAux_Pairwise.Mahalanobis_Dist(mCalc, colMeans(mCalc))

    ret_ <- auxMBenef
    return (ret_)
  }

  #
  # Determine R score
  #
  aux_step06_RankingNMMD<-function(inPathsIDs, inDist ){

    nRow <- dim(inDist)[1]
    nCol <- dim(inDist)[2]
    auxCi <- matrix(ncol=nCol, nrow=nRow)
    #auxCi[,1] <- 1:nRow
    auxCi[,1] <- inPathsIDs
    mCalc <- inDist

    fDist <- function(i){ # Apply a function on pairwise columns
      auxCj <- sum(mCalc[i,]) / (ncol(mCalc))
      return (auxCj)
    }

    # dist
    aDist <- sapply(1:nrow(mCalc), fDist)
    auxCi[,2] <- aDist
    return(auxCi)
  }

  ncolMB <- dim(mBen_Criteria)[2]
  NMMDTOPsisBenefits <- as.matrix(mBen_Criteria )

  ncolMC <- dim(mCost_Criteria)[2]
  NMMDTOPsisCosts <- as.matrix(mCost_Criteria )

  NMMDWeiBenTOP <- vBen_weight
  NMMDWeiCostTOP <- vCost_weight

  #
  # Step 01
  #

  #Keep Path Ids
  NMMD_PATHIds <- mBen_Criteria[,1]

  NMMDTOPsisCostsTended <- aux_step01_Tendency_Cnv(NMMDTOPsisCosts)
  NMMDWeiBenTOPall <- aux_step01_Tendend_Weigths(NMMDWeiBenTOP, NMMDWeiCostTOP)
  NMMDTOPsisBenefits <- cbind(NMMDTOPsisBenefits, NMMDTOPsisCostsTended)
  NMMDWeiBenTOP <- NMMDWeiBenTOPall
  stopifnot(NMMDTOPsisBenefits != NULL)

  #
  # Step 02
  #
  #naCol <- dim(NMMDTOPsisBenefits)[2]
  #print(NMMDTOPsisBenefits[,2:naCol])
  #if (!fAux_Do_Mahalanobis(NMMDTOPsisBenefits[,2:naCol])){
  #	print("Doing Normalization")
  NMMDTOPsisBenefitsTOP <- aux_step02_Normalization(NMMDTOPsisBenefits)
  #}

  #
  # Step 03
  #
  NMMDTOPsisBenefitsTOP <- aux_step03_Weight_of_Norm_matrix(NMMDTOPsisBenefitsTOP, NMMDWeiBenTOP)

  # Step 04 - Ideal Solutions

  # Step 05 - Distance to Ideal

  NMMDDistBen_Dia <- aux_step05_distance_to_idealNMMD( NMMDTOPsisBenefitsTOP)

  #Step 06

  NMMDtopsisRanking <- aux_step06_RankingNMMD(NMMD_PATHIds, NMMDDistBen_Dia$distance)
  #NMMDtopsisRanking

  #print(NMMDtopsisRanking)
  #print("NMMD")
  #print(NMMDtopsisRanking[order(-NMMDtopsisRanking[,2])])
  #print("\n")
  return(NMMDtopsisRanking[order(-NMMDtopsisRanking[,2]),])

}


       #par(mfrow=c(2,1), mgp = c(1.5, 0.5, 0),tck=0.02)

       V<- 4.66*10^(-13)

       #V是基础弯月面体积

       vne1<- 0.5*q[1]/fv + 4.66*10^(-13)

       vne2<- (3*10^(-12))*(1-k)/fv

       #vne是体积增量

       fv<-c(1:200)
       q<-c(1.5*10^(-12),8.33*10^(-12),16.67*10^(-12),
            1.5/60*10^(-12),27/60*10^(-12),54/60*10^(-12),3*10^(-12))
       k<-c(0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8)

       #fv,q,k分别为电压频率，流量和占空比

       plot(fv,vne1, mgp = c(1.5, 0.5, 0),tck=0.02,type="b",pch=1,cex=0.6,lwd=1.5,lty=2,xlab = expression(italic(f["v"])(Hz)),
            ylab = expression (italic(V["ne"]+V["m"](m^3))),xlim=c(0,200),ylim=c(4.66*10^(-13), 9*10^(-13)))

       coll<-rainbow(7)
       pcc<-c(1,2,3,4,5,6,7)

       for(i in 2:7){
         lines(fv,(0.5*q[i]/fv + 4.66*10^(-13)),col=coll[i],pch=pcc[i],cex=0.6, type="b",lwd=1.5,
               lty=2)
       }

       abline(h=(V+0.05*V),col="red",lwd=1.5,lty=4)
       abline(h=(V+0.1*V),col="red",lwd=1.5,lty=4)
       abline(h=(V+0.2*V),col="red",lwd=1.5,lty=4)
       abline(h=(V+0.6*V),col="red",lwd=1.5,lty=4)

       legend("topright",c("1.5nl/s","8.33nl/s","16.67nl/s",
                           "1.5nl/min","27nl/min","54nl/min",
                           "180nl/min"),col=c("black",
         coll[2],coll[3],coll[4],coll[5],coll[6],coll[7]),
         inset = .02,pch=c(1,2,3,4,5,6,7),bty = "n",lty=2,lwd=1.5,cex=0.8)