# -------------------------------------------
#  App Title: Benford's Law and Data Examples
#     Author: Jimmy Doi
# -------------------------------------------

library(RColorBrewer)
library(rvest)
library(xml2)

shinyUI(navbarPage("Benford's Law: Data Examples",

  #open tabPanel#1
  #############################################################################
  ## tabPanel: Census Data I                                                 ##
  #############################################################################
  tabPanel("(1) Census Data I",
    fluidPage(
     tags$head(tags$link(rel = "icon", type = "image/x-icon", href = "https://webresource.its.calpoly.edu/cpwebtemplate/5.0.1/common/images_html/favicon.ico")),

    # Give the page a title
    h3("(1) Benford's Law: US Census Data I",HTML("–"),"Population Estimates"),

    div("Note: Please adjust width of browser if only one column is visible.",br(),
        HTML("[Click here for another Shiny app on Benford's Law]"),
        style = "font-size: 9pt;color:teal"),br(),

    p("The first-digit distribution of many US Census variables is known to closely follow",
      HTML("Benford's Law."),
    "We will consider several census variables available from",
     HTML("
          County Totals Dataset: Population, Population Change and Estimated
          Components of Population Change."),
     "The app will apply a goodness of fit test of the observed frequencies of first-digits
     for the selected variable. The variables under consideration are: Annual Resident Total Population
     Estimate (2013 to 2016), Annual Births (2013 to 2016), Annual Deaths (2013 to 2016)."),

    p("Note: It may be the case that some variables do not sufficiently adhere
    to Benford's Law according to the goodness of fit test.
    However bear in mind that relatively small deviations from what is expected
    can lead to a small P-value for the test due to a large sample size. Nevertheless,
    it is still interesting to see that, for any census variable from this app, the observed
    proportions of first-digits are not uniformly equal to 1/9 as one might expect and that Benford's
    Law can at least serve as a rough approximation."),

    HTML("
"),

    # Generate a row with a sidebar
    fluidRow(
      column(4,wellPanel(

        selectInput("sel.var.cens", label = h5("Select Census Variable:"),
                    choices = list("Population Est. 2016" = 16,
                                   "Population Est. 2015" = 15,
                                   "Population Est. 2014" = 14,
                                   "Population Est. 2013" = 13,
                                   "Births, 2016" = 30,
                                   "Births, 2015" = 29,
                                   "Births, 2014" = 28,
                                   "Births, 2013" = 27,
                                   "Deaths, 2016" = 37,
                                   "Deaths, 2015" = 36,
                                   "Deaths, 2014" = 35,
                                   "Deaths, 2013" = 34
                                   ),
                    selected = 16),

        br(), br(), br(), br(),

        div("Shiny app by", a(href="http://statweb.calpoly.edu/jdoi/",target="_blank",
        "Jimmy Doi"),align="right", style = "font-size: 8pt"),

        div("Base R code by", a(href="http://statweb.calpoly.edu/jdoi/",target="_blank",
        "Jimmy Doi"),align="right", style = "font-size: 8pt"),

        div("Shiny source files:", a(href="https://gist.github.com/calpolystat/94fe941ab0d8a4f36d8b",
        target="_blank","GitHub Gist"),align="right", style = "font-size: 8pt"),

        div(a(href="http://www.statistics.calpoly.edu/shiny",target="_blank",
        "Cal Poly Statistics Dept Shiny Series"),align="right", style = "font-size: 8pt")
          ) # Close wellPanel
        ), # Close column-4

      column(8,
        div(span("Data from:",style="text-align:
            right;font-size:11pt;display:inline-block"),
        HTML("
          County Totals Dataset"),
        style="font-size:11pt;"),

        div(span("Total Number of Counties in Data Set: ",style="text-align:
                  right;font-size:11pt;display:inline-block"),
            span(textOutput("comp.rows.cens"),style="text-align: right;font-size:11pt;
                 display:inline-block;color:#5C8AE6;font-weight:normal")),

        div(span("Census Variable: ",style="text-align:
            right;font-size:11pt;display:inline-block"),
            span(textOutput("out.var.cens"),style="text-align: right;font-size:11pt;
            display:inline-block;color:#5C8AE6;font-weight:normal")
            ),

        HTML("
"),

        fluidRow(
          column(4,
           p(tags$b("Goodness of Fit Test:")),
           verbatimTextOutput("goodness.cens")
           ),
          column(8,
            plotOutput("cens.pmf")
            )#closes column-8
           ),#closes fluidRow
          HTML("
")
     ) #closes column-8
    ) #closes fluidRow
   ) #closes fluidPage
  ), #close tabPanel#1

  #open tabPanel#2
  #############################################################################
  ## tabPanel: Census Data II                                                ##
  #############################################################################
  tabPanel("(2) Census Data II",
    fluidPage(

    # Give the page a title
    h3("(2) Benford's Law: US Census Data II",HTML("–"),"QuickFacts"),

    div("Note: Please adjust width of browser if only one column is visible.",br(),
        HTML("[Click here for another Shiny app on Benford's Law]"),
        style = "font-size: 9pt;color:teal"),br(),

    p("The first-digit distribution of many US Census variables is known to closely follow",
      HTML("Benford's Law."),
    "We will consider several census variables available from",
     HTML("US Census State & County QuickFacts."),
     "The app will apply a goodness of fit test of the observed frequencies of first-digits
     for the selected variable. The variables under consideration are:
      Housing Units (2013),
      Households (2008-12),
      Veterans (2008-12),
      Nonemployer Establishments (2012),
      *Private Nonfarm Establishments (2012),
      *Private Nonfarm Employment (2012),
      *Retail Sales (2007)."),

    p("*A small fraction (less than 2%) of the 3143 counties had entries of
        zero for the variables listed with an asterisk. For these cases, only the non-zero
        values were used for the goodness of fit test.",style="font-style:normal;font-size:9pt"),

    p("Note: It may be the case that some variables do not sufficiently adhere
    to Benford's Law according to the goodness of fit test.
    However bear in mind that relatively small deviations from what is expected
    can lead to a small P-value for the test due to a large sample size. Nevertheless,
    it is still interesting to see that, for any census variable from this app, the observed
    proportions of first-digits are not uniformly equal to 1/9 as one might expect and that Benford's
    Law can at least serve as a rough approximation."),

    HTML("
"),

    # Generate a row with a sidebar
    fluidRow(
      column(4,wellPanel(

        selectInput("sel.var.quick", label = h5("Select Census Variable:"),
                    choices = list("Housing Units, 2013" = 25,
                                   "Households, 2008-12" = 29,
                                   "Veterans, 2008-12" = 23,
                                   "Nonemployer Est., 2012" = 37,
                                   "*Private Nonfarm Est., 2012" = 34,
                                   "*Private Nonfarm Emp., 2012" = 35,
                                   "*Retail Sales, 2007 ($1000)" = 47),
                    selected = 25),

        br(), br(), br(), br(),

        div("Shiny app by", a(href="http://statweb.calpoly.edu/jdoi/",target="_blank",
        "Jimmy Doi"),align="right", style = "font-size: 8pt"),

        div("Base R code by", a(href="http://statweb.calpoly.edu/jdoi/",target="_blank",
        "Jimmy Doi"),align="right", style = "font-size: 8pt"),

        div("Shiny source files:", a(href="https://gist.github.com/calpolystat/94fe941ab0d8a4f36d8b",
        target="_blank","GitHub Gist"),align="right", style = "font-size: 8pt"),

        div(a(href="http://www.statistics.calpoly.edu/shiny",target="_blank",
        "Cal Poly Statistics Dept Shiny Series"),align="right", style = "font-size: 8pt")
          ) # Close wellPanel
        ), # Close column-4

      column(8,
        div(span("Data from:",style="text-align:
            right;font-size:11pt;display:inline-block"),
            HTML("US Census State & County QuickFacts"),
            style="font-size:11pt;"),

        div(span("Total Number of Counties in Data Set: ",style="text-align:
                  right;font-size:11pt;display:inline-block"),
            span(textOutput("comp.rows.quick"),style="text-align: right;font-size:11pt;
                 display:inline-block;color:#5C8AE6;font-weight:normal")),

        div(span("Census Variable: ",style="text-align:
            right;font-size:11pt;display:inline-block"),
            span(textOutput("out.var.quick"),style="text-align: right;font-size:11pt;
            display:inline-block;color:#5C8AE6;font-weight:normal")
            ),

        HTML("
"),

        fluidRow(
          column(4,
           p(tags$b("Goodness of Fit Test:")),
           verbatimTextOutput("goodness.quick")
           ),
          column(8,
            plotOutput("quick.pmf")
            )#closes column-8
           ),#closes fluidRow
          HTML("
")
     ) #closes column-8
    ) #closes fluidRow
   ) #closes fluidPage
  ), #close tabPanel#2

  #open tabPanel#3
  #############################################################################
  ## tabPanel: Stock Exchange                                                ##
  #############################################################################
  tabPanel("(3) US Stock Markets",
           fluidPage(

             # Give the page a title
             h3("(3) Benford's Law: US Stock Markets"),

             div("Note: Please adjust width of browser if only one column is visible.",
                 style = "font-size: 9pt;color:teal"),br(),

             p("After a given day of trading, if we consider the first-digit of the
               volume of shares traded for each listed company in the New York Stock Exchange,
               the corresponding distribution will closely follow",
               HTML("Benford's Law."),
               "Does the first-digit distribution for other variables such as a stock's closing cost
               closely follow Benford's Law? And does the specific stock market have any influence on the
               first-digit distribution of market variables?"),

             p("This app will download information from the",
               span(HTML("Wall Street Journal website")),
               "from the",span("most recent end of day market data.",style="font-weight:bold"),
               "The data will be based on various
               market variables for all
               companies listed in one of four stock markets.
               The app will apply a goodness of fit test of the observed frequencies of first-digits
               for the selected variable in the specified stock market."),

             p("Note: It may be the case that some stock price variables do not sufficiently adhere
               to Benford's Law according to the goodness of fit test.
               However bear in mind that relatively small deviations from what is expected
               can lead to a small P-value for the test due to a large sample size. Nevertheless,
               it is still interesting to see that, for any variable in this app, the observed
               proportions of first-digits are not uniformly equal to 1/9 as one might expect and that Benford's
               Law can at least serve as a rough approximation."),

             HTML("
"),

             # Generate a row with a sidebar
             fluidRow(
               column(4,wellPanel(

                 selectInput("stock", label = h5("Select Stock Market:"),
                             choices = c("New York Stock Exchange",
                                         "Nasdaq Stock Market",
                                         "Nasdaq Capital Market",
                                         "NYSE MKT Stock Exchange"),
                             selected = "New York Stock Exchange"), br(),

                 selectInput("sel.var", label = h5("Select Variable"),
                             choices = list("Volume of Shares Traded" = 9,
                                            "Opening Price" = 3,
                                            "High Price" = 4,
                                            "Low Price" = 5,
                                            "Closing Price" = 6),
                             selected = 9),

                 br(), br(), br(), br(),

                 div("Shiny app by", a(href="http://statweb.calpoly.edu/jdoi/",target="_blank",
                                       "Jimmy Doi"),align="right", style = "font-size: 8pt"),

                 div("Base R code by", a(href="http://statweb.calpoly.edu/jdoi/",target="_blank",
                                         "Jimmy Doi"),align="right", style = "font-size: 8pt"),

                 div("Shiny source files:", a(href="https://gist.github.com/calpolystat/f4475cbfe4cc77cef168",
                                              target="_blank","GitHub Gist"),align="right", style = "font-size: 8pt"),

                 div(a(href="http://www.statistics.calpoly.edu/shiny",target="_blank",
                       "Cal Poly Statistics Dept Shiny Series"),align="right", style = "font-size: 8pt")
               ) # Close wellPanel
               ), # Close column-4

               column(8,

                      div(span(imageOutput("USA.icon.stock",height=24),style="vertical-align: middle;display:inline-block"),
                          span(HTML(" Currency:"),style="text-align:left;
                               font-size:11pt;vertical-align: middle;display:inline-block"),
                          span(HTML(" "),style="text-align:left;font-size:11pt;vertical-align: middle;display:inline-block"),
                          span(textOutput("USA.curr.stock"),style="text-align:left;font-size:11pt;color:#5C8AE6;
                               display:inline-block;vertical-align: middle"),
                          span(HTML("         "),style="text-align: right;vertical-align: middle;font-size:11pt"),
                      span("End of Day Market Data from: ",style="text-align:
                      right;font-size:11pt;vertical-align: middle;display:inline-block"),
                          span(HTML(" "),style="text-align: right;vertical-align: middle;font-size:11pt"),
                          span(textOutput("comp.date"),style="text-align: right;vertical-align: middle;font-size:11pt;
                      display:inline-block;color:#5C8AE6;font-weight:normal"),
                          style="text-align: right;font-size:0pt;display:inline-block;"),br(),

                      div(span("Market Source: ",style="text-align:
                      right;font-size:11pt;display:inline-block"),
                          span(textOutput("short.stock"),style="text-align: right;font-size:11pt;
                     display:inline-block;color:#5C8AE6;font-weight:normal"),
                          span(HTML("    "),style="text-align: right;font-size:11pt"),
                          span("Total Number of Stocks in Data Set: ",style="text-align:
                      right;font-size:11pt;display:inline-block"),
                          span(textOutput("comp.rows"),style="text-align: right;font-size:11pt;
                     display:inline-block;color:#5C8AE6;font-weight:normal")),

                      HTML("
"),

                      fluidRow(
                        column(4,
                               p(tags$b("Goodness of Fit Test:")),
                               div(span(textOutput("out.var"),style="text-align: right;font-size:11pt;
                      display:inline-block;color:#5C8AE6;font-weight:normal;text-align: center")),
                               br(),
                               verbatimTextOutput("goodness")
                        ),
                        column(8,
                               plotOutput("stock.pmf")
                        )#closes column-8
                      ),#closes fluidRow
                      HTML("
")
               ) #closes column-8
             ) #closes fluidRow
             ) #closes fluidPage
               ), #close tabPanel#3

  #open tabPanel#4
  #############################################################################
  ## tabPanel: World Stock Exchange                                          ##
  #############################################################################
  tabPanel("(4) World Stock Markets",
           fluidPage(

             # Give the page a title
             h3("(4) Benford's Law: World Stock Markets"),

             div("Note: Please adjust width of browser if only one column is visible.",br(),
                 HTML("[Click here for another Shiny app on Benford's Law]"),
                 style = "font-size: 9pt;color:teal"),br(),

             p("For the New York Stock Exchange (see previous tab),
              we found that the first-digit distribution of stock prices
              closely follows",
               HTML("Benford's Law."),
               "Does Benford's Law also apply to the first-digit distribution for stock prices from other world markets?
               Does currency have any influence on the
               first-digit distribution of market variables?"),

             p("This app will download market data from the",
               span(HTML("Investing.com website. ")),
               "For the selected stock market, if trading is ", span("active ",style="font-weight:bold"),
               "at the point of data access,
                the results will be based on the ", span("most current market data.",style="font-weight:bold"),
               "If the market is closed at point of access, then all information will be based on the most recent
                end of day market data. The app will apply a goodness of fit test of the observed frequencies of first-digits
               for the selected variable in the specified stock market."),

             p("Note: It may be the case that some stock price variables do not sufficiently adhere
               to Benford's Law according to the goodness of fit test.
               However bear in mind that relatively small deviations from what is expected
               can lead to a small P-value for the test due to a large sample size. Nevertheless,
               it is still interesting to see that, for any variable in this app, the observed
               proportions of first-digits are not uniformly equal to 1/9 as one might expect and that Benford's
               Law can at least serve as a rough approximation."),

             HTML("
"),

             # Generate a row with a sidebar
             fluidRow(
               column(4,wellPanel(

                 selectInput("w.stock", label = h5("Select World Market:"),
                             choices = c("Australia",
                                         "Canada",
                                         "China",
                                         "Denmark",
                                         "England",
                                         "France",
                                         "Germany",
                                         "India",
                                         "Japan",
                                         "Korea",
                                         "Netherlands",
                                         "Poland",
                                         "Sri Lanka",
                                         "Switzerland",
                                         "Turkey"),
                             selected = "Australia"), br(),

                 selectInput("w.sel.var", label = h5("Select Variable:"),
                             choices = list("Most Recent Price" = 3,
                                            "High Price" = 4,
                                            "Low Price" = 5),
                             selected = 3),

                 br(), br(), br(), br(),

                 div("Shiny app by", a(href="http://statweb.calpoly.edu/jdoi/",target="_blank",
                                       "Jimmy Doi"),align="right", style = "font-size: 8pt"),

                 div("Base R code by", a(href="http://statweb.calpoly.edu/jdoi/",target="_blank",
                                         "Jimmy Doi"),align="right", style = "font-size: 8pt"),

                 div("Shiny source files:", a(href="https://gist.github.com/calpolystat/94fe941ab0d8a4f36d8b",
                                              target="_blank","GitHub Gist"),align="right", style = "font-size: 8pt"),

                 div(a(href="http://www.statistics.calpoly.edu/shiny",target="_blank",
                       "Cal Poly Statistics Dept Shiny Series"),align="right", style = "font-size: 8pt")
               ) # Close wellPanel
               ), # Close column-4

               column(8,

                      div(span(imageOutput("w.icon.stock",height=24),style="vertical-align: middle;display:inline-block"),
                          span(HTML(" Currency:"),style="text-align:left;
                               font-size:11pt;vertical-align: middle;display:inline-block"),
                          span(HTML(" "),style="text-align:left;font-size:11pt;vertical-align: middle;display:inline-block"),
                          span(textOutput("w.curr.stock"),style="text-align:left;font-size:11pt;color:#5C8AE6;
                               display:inline-block;vertical-align: middle"),
                          span(HTML("          "),
                               style="text-align: left;font-size:11pt;vertical-align: middle"),
                          span("Access Point: ",style="text-align:
                               left;font-size:11pt;display:inline-block;vertical-align: middle"),
                          span(HTML(" "),style="text-align: left;font-size:11pt"),
                          span(textOutput("w.time.stock"),style="text-align: left;font-size:11pt;
                               display:inline-block;color:#5C8AE6;font-weight:normal;vertical-align: middle"),
                          style="text-align: right;font-size:0pt;display:inline-block;"),br(),

                      div(span("Market Status:",style="text-align:
                               left;font-size:11pt;display:inline-block;vertical-align: middle"),
                          span(textOutput("w.mkt.status"),style="text-align: left;font-size:11pt;
                               display:inline-block;color:red;font-weight:normal;vertical-align: middle"),
                          span(HTML("    "),style="text-align: left;font-size:11pt;vertical-align: middle"),
                          span("Source: ",style="text-align:
                               left;font-size:11pt;display:inline-block;vertical-align: middle"),
                          span(textOutput("w.short.stock"),style="text-align: left;font-size:11pt;
                               display:inline-block;color:#5C8AE6;font-weight:normal;vertical-align: middle"),
                          span(HTML("    "),style="text-align: left;font-size:11pt;vertical-align: middle"),
                          span("Total Stocks: ",style="text-align:
                               left;font-size:11pt;display:inline-block;vertical-align: middle"),
                          span(textOutput("w.comp.rows"),style="text-align: left;font-size:11pt;
                          display:inline-block;color:#5C8AE6;font-weight:normal;vertical-align: middle")
                          ),

                      HTML("
"),

                      fluidRow(
                        column(4,
                               p(tags$b("Goodness of Fit Test:")),
                               div(span(textOutput("w.out.var"),style="text-align: right;font-size:11pt;
                                        display:inline-block;color:#5C8AE6;font-weight:normal;text-align: center")),
                               br(),
                               verbatimTextOutput("w.goodness")
                               ),
                        column(8,
                               plotOutput("w.stock.pmf")
                        )#closes column-8
                      ),#closes fluidRow
                      HTML("
")
                          ) #closes column-8
                      ) #closes fluidRow
             ) #closes fluidPage
               ) #close tabPanel#4

  ) #closes navbarPage
) #closes shinyUI


subroutine delseg(delsgs,ndel,nadj,madj,npd,x,y,ntot,nerror)

# Output the endpoints of the line segments joining the
# vertices of the Delaunay triangles.
# Called by master.

implicit double precision(a-h,o-z)
logical value
dimension nadj(-3:ntot,0:madj), x(-3:ntot), y(-3:ntot)
dimension delsgs(6,ndel)

# For each distinct pair of points i and j, if they are adjacent
# then put their endpoints into the output array.
npd = ntot-4
kseg = 0
do i = 2,npd {
        do j = 1,i-1 {
                call adjchk(i,j,value,nadj,madj,ntot,nerror)
		if(nerror>0){
                    return
                }
                if(value) {
			kseg = kseg+1
			if(kseg > ndel) {
				nerror = 14
				return
			}
			delsgs(1,kseg) = x(i)
			delsgs(2,kseg) = y(i)
			delsgs(3,kseg) = x(j)
			delsgs(4,kseg) = y(j)
			delsgs(5,kseg) = i
			delsgs(6,kseg) = j
                }
        }
}
ndel = kseg

return
end


# In the integrated analysis, all of the data components share the same mean Linf parameter ("mu.Linf").
# The tagging and otolith components include a random Linf parameter, so also share the same "sig.Linf";
# whereas the length-frequency component does not include a random Linf (thus no "sig.Linf") since
# the data are for young fish that are not near their asymptotic length.

# There is no information about the a0 parameter from the tagging data, so this parameter is estimated
# from the otolith and length-frequency data sets

# The variance parameters are assumed to be unique for each data set

# The length-frequency (LF) component assumes that a model decomposition has already been applied to the
# raw data, such that the dataset input to the growth analysis is the lengths and ages corresponding to
# the modes in the LF data.

# - param.f contains the parameters of the growth function, f (note L=Linf*f, so param.f does not include Linf)
#   (e.g. param.f={k} for the standard VB model; param.f={k,u,w} for VB with seasonal growth)
# - npf = number of parameters for the growth function, f; i.e., length of param.f
#   (e.g. npf=1 for standard VB; npf=3 for VB with seasonal growth)
# - param.A contains the parameters for the distribution of A (e.g. log normal in our example)
# - npA = number of parameters for distribution of A, i.e., length of param.A (e.g. npA=2 for log normal)
# - param contains all of the parameters being estimated, including the variance parameters; note that
#   for the tagging component, we allow measurement error in length to be different for scientists vs fishermen
#   since many of our recaptures were measured by fishermen, hence the 2 variance parameters sig.s and sig.f
#   (e.g., param = {mu.Linf, sig.Linf, param.f, param.A, sig.s, sig.f, a0, sig.oto, sig.lf}

# 1) Otolith negative log-likelihood component:
# - assumes data is a matrix with columns: age, length

logl.oto.f <- function(param, npf, npA, otodat)
{
  #print(param)
  # Get data:
	age <- otodat[, 1]
	len <- otodat[, 2]
  n <- nrow(otodat)

	# Get param:
	mu.L <- param[1]
	sig.L <- param[2]
	param.f <- param[3:(npf+2)]
	a0 <- param[2+npf+npA+3]
	sig.oto <- param[2+npf+npA+4]

	grwth <- growth.ssnl.f(age - a0, param.f, age)
	exp.len <- mu.L * grwth
	var.len <- (grwth * sig.L)^2 + sig.oto^2
	neglogl <- 1/2 * sum(log(2 * pi * var.len) + ((len - exp.len)^2)/var.len)

	#print(neglogl)
	return(neglogl)
}

#2) Length-frequency (LF) negative log-likelihood component:
# - assumes data is in a matrix with columns: age, mean.mode, se.mode

logl.lf.f <- function(param, npf, npA, lfdat)
{
	# Get data:
	age <- lfdat[, 1]
	mu.mode <- lfdat[, 2]
	se.mode <- lfdat[, 3]
	n <- nrow(lfdat)

	# Get parameters:
	mu.L <- param[1]
	param.f <- param[3:(npf+2)]
	a0 <- param[2+npf+npA+3]
	#a0 = 0 # Note: Uncomment this to remove the effect of fitting a0.
	sig.lf <- param[2+npf+npA+5]

	grwth <- growth.ssnl.f(age - a0, param.f, age)
	exp.mode <- mu.L * grwth
	var.mode <- sig.lf^2 + se.mode^2
	neglogl <- 1/2 * sum(log(2 * pi * var.mode) + ((mu.mode - exp.mode)^2)/var.mode)
	#print(param)
	#print(neglogl)
	return(neglogl)
}

#3) Tagging negative log-likelihood component:
# - code and description found in file 'tag lkhd.r'
#source("/Users/stephenscherrer/Google Drive/Weng Lab/Data/Bottomfish/Okamoto's 1990s Mark Recapture Data/src/Laslett Functions/tag_lkhd.r")

# JOINT LIKELIHOOD:
joint.logl.f <- function(param,npf,npA,tagdat,otodat,lfdat,wt.oto=1,wt.tag=1,wt.lf=1)
{
  neglogl.tag<- 0
  neglogl.oto<- 0
  neglogl.lf<- 0
  if(wt.tag>0)  neglogl.tag <- logl.ssnl.f(param,npf,npA,tagdat)
  if(wt.oto>0)  neglogl.oto <- logl.oto.f(param,npf,npA,otodat)
  if(wt.lf>0) neglogl.lf <- logl.lf.f(param,npf,npA,lfdat)
  neglogl <- wt.tag*neglogl.tag + wt.oto*neglogl.oto + wt.lf*neglogl.lf
  #print(param)
  #print(c(neglogl.tag, neglogl.oto, neglogl.lf, neglogl))
  return(neglogl)
}

# where...

# logl.ssnl.f is for tagging data and is found in "tag lkhd.r" (along with all
# necessary associated functions)


  subset.biomass = function(Q, R, nodes, type="mature.only", fraction11 = 0.2) {
    out = NULL
    names(Q) = nodes
    names(R) = nodes

    Q[!is.finite(Q)] = 0
    R[!is.finite(R)] = 0

    if (type =="historic") {
      tot = fraction11 * (Q["imm.11"] + Q["imm.sm.11"] + Q["CC3to4.11"] + Q["CC5.11"]) +
                  (Q["imm.12"] + Q["imm.sm.12"] + Q["CC3to4.12"] + Q["CC5.12"]) +
                  (                               Q["CC3to4.13"] + Q["CC5.13"])

      err = sqrt( fraction11^2 * (R["imm.11"]^2 + R["imm.sm.11"]^2 + R["CC3to4.11"]^2 + R["CC5.11"]^2) +
                                  R["imm.12"]^2 + R["imm.sm.12"]^2 + R["CC3to4.12"]^2 + R["CC5.12"]^2 +
                                  R["CC3to4.13"]^2 + R["CC5.13"]^2
                )
    } else if (type =="mature.only") {
      tot = fraction11 * (  Q["CC3to4.11"] + Q["CC5.11"]) +
                         (  Q["CC3to4.12"] + Q["CC5.12"]) +
                         (  Q["CC3to4.13"] + Q["CC5.13"])

      err = sqrt( fraction11^2 * (  R["CC3to4.11"]^2 + R["CC5.11"]^2) +
                                    R["CC3to4.12"]^2 + R["CC5.12"]^2 +
                                    R["CC3to4.13"]^2 + R["CC5.13"]^2
                )
    }

    names(tot) = NULL
    return( c(tot,err) )
  }


#' compute Area under the Curve (AUC)
#' @param target the target
#' @param score the score, the higher the score the less targets
#' @importFrom data.table setkey shift data.table
auc <- function(target, score) {
  a = data.table::data.table(target, score = -score)
  data.table::setkey(a, score)
  a1 = a[,.(h = sum(target), w = .N), score]

  a1[,height:=cumsum(h)/sum(h)]
  a1[,ctot:=cumsum(w)/sum(w)]

  a1[,lag_height:=shift(height,1)]
  a1[1,lag_height := 0]

  a1[,area := (lag_height + height)*w/sum(w)/2]
  a1[,sum(area)]
}


#!/usr/bin/Rscript
library(psycho)  # used for SDT analysis
library(ggplot2)
library(dplyr)
library(lmerTest)

default_filename <- 'data/cat_20190321_turtle_feedback.csv'

# accept a filename as an optional command-line argument
args <- commandArgs(trailingOnly=T)
if (length(args) < 1) {
    args <- c(default_filename)
}

# estimates SDT measures of sensitivity & bias for each subject/condition
runSDT <- function(data) {
    # count the number of correct/incorrect trials for each condition
    nIncorrect <- aggregate(wasCorrect ~ subject+condition+isTarget+isFoil+isOld, data,
                            function(x) sum(x==0))$wasCorrect
    data <- aggregate(wasCorrect ~ subject+condition+isTarget+isFoil+isOld, data, sum)
    data$nCorrect <- data$wasCorrect
    data$nIncorrect <- nIncorrect

    # group the old/new stimuli by subject and condition
    data <- merge(subset(data, isOld==1), subset(data, isOld==0), sort=F,
                  by=c('subject', 'condition', 'isTarget', 'isFoil'),
                  suffixes=c('.old', '.new'))
    write.csv(data, 'sdt.csv')

    # get sensitivty & bias for each subject on each condition
    indices <- psycho::dprime(data$nCorrect.old, data$nIncorrect.new,
                              data$nIncorrect.old, data$nCorrect.new)
    sdtData <- cbind(data, indices)

    # analyze sensitivity (dprime)
    writeLines('\n\nSensitivity (dprime)')
    print(summary(lmer(dprime ~ condition*isTarget*isFoil + (1|subject), data=sdtData)))

    # analyze bias (c)
    writeLines('\n\nBias (C)')
    print(summary(lmer(c ~ condition*isTarget*isFoil + (1|subject), data=sdtData)))
    return(sdtData)
}

# analyze data from each file
for (i in 1:length(args)) {
    filename <- args[i]
    writeLines(args[i])

    memData <- subset(read.csv(filename, header=T), task=='test')
    memData$condition <- as.factor(memData$condition)
    memData$isTarget <- as.factor(memData$isTarget)
    memData$isFoil <- as.factor(memData$isFoil)
    learnData <- subset(read.csv(filename, header=T), task=='learn')
    endAcc <- aggregate(wasCorrect~subject, learnData, function (trials) mean(tail(trials, 20)))
    testRT <- aggregate(RT ~ subject, memData, mean)
    learnRT <- aggregate(RT ~ subject, learnData, mean)
    writeLines(sprintf('Total number of subjects: %d', length(unique(memData$subject))))

    # exclude subjects with high RT or low learning accuracy
    writeLines(sprintf('Mean Test RT+3SD: %f', (mean(memData$RT)+3*sd(memData$RT)) / 1000))
    writeLines(sprintf('Mean Learning RT+3SD: %f', (mean(learnData$RT)+3*sd(learnData$RT)) / 1000))
    excluded = unique(c(learnRT$subject[learnRT$RT > (mean(learnRT$RT) + 3*sd(learnRT$RT))],
                        testRT$subject[testRT$RT > (mean(testRT$RT) + 3*sd(testRT$RT))],
                        endAcc$subject[endAcc$wasCorrect < 0.85]))
    memData <- memData[!(memData$subject %in% excluded),]
    learnData <- learnData[!(learnData$subject %in% excluded),]
    writeLines(sprintf('After exclusion: %d', length(unique(memData$subject))))

    print(aggregate(subject~condition, memData, function (s) length(unique(s))))

    writeLines('\n\nRT')
    print(summary(lmer(RT ~ condition*isTarget*isFoil + (1|subject), data=memData)))

    hits <- subset(memData, isOld==1)
    FAs  <- subset(memData, isOld==0)
    FAs$wasCorrect <- 1-FAs$wasCorrect  # we want FA rate, not CR rate

    writeLines('\n\nHits')
    print(summary(glmer(wasCorrect ~ condition*isTarget*isFoil + (1|subject),
                        data=hits, family=binomial(link='logit'))))
    writeLines('\n\nFAs')
    print(summary(glmer(wasCorrect ~ condition*isTarget*isFoil + (1|subject),
                        data=FAs, family=binomial(link='logit'))))

    sdtData <- runSDT(memData)
    writeLines('\n\n\n\n\n')
    write.csv(sdtData, 'sdt.csv')

    # plot interaction plots
    png('feedback_RT.png', height=1000, width=1000)
    # remove individual differences (for within-subject studies)
    subjAvgs <- memData %>% group_by(subject) %>% summarise(subjAvg=mean(RT))
    condAvgs <- memData %>% group_by(condition) %>% summarise(condAvg=mean(RT))
    memData$RT <- apply(memData, 1, function (x) as.numeric(x['RT'])
                        - subset(subjAvgs, subject==as.numeric(x['subject']))$subjAvg
                        + subset(condAvgs, condition == x['condition'])$condAvg)
    print(ggplot(memData %>% group_by(condition, isTarget, isFoil) %>%
                 summarise(N=length(RT), rt=mean(RT), sd=sd(RT), ci=(1.96*sd/sqrt(N)))) +
          aes(x=interaction(isFoil, isTarget), y=rt, fill=isTarget) +
          geom_col(size=2) + theme_classic() + ylab('Reaction Time') +
          geom_errorbar(aes(ymin=rt-ci, ymax=rt+ci), width=.2) +
          facet_wrap(~condition, strip.position='bottom') +
          aes(x=interaction(isFoil, isTarget), y=rt, fill=isTarget) +
          geom_col(size=2) + theme_classic() + ylab('Reaction Time') +
          geom_errorbar(aes(ymin=rt-ci, ymax=rt+ci), width=.2) +
          facet_wrap(~condition, strip.position='bottom') +
          scale_x_discrete(labels=c('Neither', 'Not Learned', 'Learned', 'Both')) +
          theme(axis.text=element_text(size=20),
                axis.title.x=element_blank(),
                axis.title.y=element_text(size=36, margin=margin(r=0.5, unit='cm')),
                legend.position='none',
                strip.background=element_blank(),
                strip.placement='outside',
                strip.text=element_text(size=36),
                plot.margin=margin(t=1, b=1, unit='cm')))
    dev.off()

    png('feedback_hits.png', height=1000, width=1000)
    # remove individual differences (for within-subject studies)
    subjAvgs <- hits %>% group_by(subject) %>% summarise(subjAvg=mean(wasCorrect))
    condAvgs <- hits %>% group_by(condition) %>% summarise(condAvg=mean(wasCorrect))
    hits$wasCorrect <- apply(hits, 1, function (x) as.numeric(x['wasCorrect'])
                             - subset(subjAvgs, subject==as.numeric(x['subject']))$subjAvg
                             + subset(condAvgs, condition == x['condition'])$condAvg)
    print(ggplot(hits %>% group_by(condition, isTarget, isFoil) %>%
                 summarise(N=length(wasCorrect), hits=mean(wasCorrect),
                           sd=sd(wasCorrect), ci=(1.96*sd/sqrt(N)))) +
          aes(x=interaction(isFoil, isTarget), y=hits, fill=isTarget) +
          geom_col(size=2) + theme_classic() + ylab('Hit Rate') +
          geom_errorbar(aes(ymin=hits-ci, ymax=hits+ci), width=.2) +
          facet_wrap(~condition, strip.position='bottom') +
          scale_x_discrete(labels=c('Neither', 'Not Learned', 'Learned', 'Both')) +
          theme(axis.text=element_text(size=20),
                axis.title.x=element_blank(),
                axis.title.y=element_text(size=36, margin=margin(r=0.5, unit='cm')),
                legend.position='none',
                strip.background=element_blank(),
                strip.placement='outside',
                strip.text=element_text(size=36),
                plot.margin=margin(t=1, b=1, unit='cm')))
    dev.off()

    png('feedback_FAs.png', height=1000, width=1000)
    # remove individual differences (for within-subject studies)
    subjAvgs <- FAs %>% group_by(subject) %>% summarise(subjAvg=mean(wasCorrect))
    condAvgs <- FAs %>% group_by(condition) %>% summarise(condAvg=mean(wasCorrect))
    FAs$wasCorrect <- apply(FAs, 1, function (x) as.numeric(x['wasCorrect'])
                            - subset(subjAvgs, subject==as.numeric(x['subject']))$subjAvg
                            + subset(condAvgs, condition == x['condition'])$condAvg)
    print(ggplot(FAs %>% group_by(condition, isTarget, isFoil) %>%
                 summarise(N=length(wasCorrect), FAs=mean(wasCorrect),
                           sd=sd(wasCorrect), ci=(1.96*sd/sqrt(N)))) +
          aes(x=interaction(isFoil, isTarget), y=FAs, fill=isTarget) +
          geom_col(size=2) + theme_classic() + ylab('FA Rate') +
          geom_errorbar(aes(ymin=FAs-ci, ymax=FAs+ci), width=.2) +
          facet_wrap(~condition, strip.position='bottom') +
          scale_x_discrete(labels=c('Neither', 'Not Learned', 'Learned', 'Both')) +
          theme(axis.text=element_text(size=20),
                axis.title.x=element_blank(),
                axis.title.y=element_text(size=36, margin=margin(r=0.5, unit='cm')),
                legend.position='none',
                strip.background=element_blank(),
                strip.placement='outside',
                strip.text=element_text(size=36),
                plot.margin=margin(t=1, b=1, unit='cm')))
    dev.off()

    png('feedback_sensitivity.png', height=1000, width=1000)
    # remove individual differences (for within-subject studies)
    subjAvgs <- sdtData %>% group_by(subject) %>% summarise(subjAvg=mean(dprime))
    condAvgs <- sdtData %>% group_by(condition) %>% summarise(condAvg=mean(dprime))
    sdtData$dprime <- apply(sdtData, 1, function (x) as.numeric(x['dprime'])
                            - subset(subjAvgs, subject==as.numeric(x['subject']))$subjAvg
                            + subset(condAvgs, condition == x['condition'])$condAvg)
    print(ggplot(sdtData %>% group_by(condition, isTarget, isFoil) %>%
                 summarise(N=length(dprime), sensitivity=mean(dprime),
                           sd=sd(dprime), ci=(1.96*sd/sqrt(N)))) +
          aes(x=interaction(isFoil, isTarget), y=sensitivity, fill=isTarget) +
          geom_col(size=2) + theme_classic() +
          geom_errorbar(aes(ymin=sensitivity-ci, ymax=sensitivity+ci), width=.2) +
          facet_wrap(~condition, strip.position='bottom') +
          aes(x=interaction(isFoil, isTarget), y=sensitivity, fill=isTarget) +
          geom_col(size=2) + theme_classic() +
          geom_errorbar(aes(ymin=sensitivity-ci, ymax=sensitivity+ci), width=.2) +
          facet_wrap(~condition, strip.position='bottom') +
          ylab(expression(paste('Sensitivity (', d*minute, ')'))) +
          scale_x_discrete(labels=c('Neither', 'Not Learned', 'Learned', 'Both')) +
          theme(axis.text=element_text(size=20),
                axis.title.x=element_blank(),
                axis.title.y=element_text(size=36, margin=margin(r=0.5, unit='cm')),
                legend.position='none',
                strip.background=element_blank(),
                strip.placement='outside',
                strip.text=element_text(size=36),
                plot.margin=margin(t=1, b=1, unit='cm')))
    dev.off()

    png('feedback_bias.png', height=1000, width=1000)
    # remove individual differences (for within-subject studies)
    subjAvgs <- sdtData %>% group_by(subject) %>% summarise(subjAvg=mean(c))
    condAvgs <- sdtData %>% group_by(condition) %>% summarise(condAvg=mean(c))
    sdtData$c <- apply(sdtData, 1, function (x) as.numeric(x['c'])
                       - subset(subjAvgs, subject==as.numeric(x['subject']))$subjAvg
                       + subset(condAvgs, condition == x['condition'])$condAvg)
    print(ggplot(sdtData %>% group_by(condition, isTarget, isFoil) %>%
                 summarise(N=length(c), bias=mean(c), sd=sd(c), ci=(1.96*sd/sqrt(N)))) +
          aes(x=interaction(isFoil, isTarget), y=bias, fill=isTarget) +
          geom_col(size=2) + theme_classic() + ylab('Bias (C)') +
          geom_errorbar(aes(ymin=bias-ci, ymax=bias+ci), width=.2) +
          facet_wrap(~condition, strip.position='bottom') +
          scale_x_discrete(labels=c('Neither', 'Not Learned', 'Learned', 'Both')) +
          theme(axis.text=element_text(size=20),
                axis.title.x=element_blank(),
                axis.title.y=element_text(size=36, margin=margin(r=0.5, unit='cm')),
                legend.position='none',
                strip.background=element_blank(),
                strip.placement='outside',
                strip.text=element_text(size=36),
                plot.margin=margin(t=1, b=1, unit='cm')))
    dev.off()
}


# Load packages -----------------------
library(dplyr)
library(tidyr)
library(purrr)
library(ggplot2)
library(cowplot)

# Functions ---------------------------
# biomass estimation based on Niiyama et al. (2010)
biomass <- function(dbh, component = "agb") {
  h <- 1 / (1 / (1.61 * dbh) + 1 / 69.) # tree height (m) from dbh (cm)
  w_s <- 0.036 * (dbh * dbh * h)^1.01 # trunk plus branch stem (kg in dry mass)
  w_l <- 1 / (1 / (0.108 * w_s^0.75) + 1 / 105) # leaves (kg in dry mass)
  w_r <- 0.023 * dbh^2.59 # coarse root (kg in dry mss)
  if (component == "agb") {
    return(w_s + w_l)
  } else if (component == "all") {
    return(w_s + w_l + w_r)
  } else if (component == "stem") {
    return(w_s)
  } else if (component == "leaf") {
    return(w_l)
  } else if (component == "root") {
    return(w_r)
  } else {
    stop("Invalid 'component'. Expected one of: 'all', 'agb', 'stem', 'leaf', 'root'")
  }
}

# common turnover rate
turnover <- function(y, z, t) {
  f <- function(rho) {
    sum(y * exp(-rho * t) - z)
  }
  df <- function(rho) {
    sum(-t * y * exp(-rho * t))
  }
  # Newton-Rapton iteration
  rho <- 0.02
  precision <- 1.0e-12 # to stop iteration
  change <- precision + 1.0

  while (change > precision) {
    rho2 <- rho - f(rho) / df(rho)
    change <- abs(rho2 - rho)
    rho <- rho2
  }
  return(rho)
}

# production rate
productivity <- function(dbh1, dbh2, w1, w2, t, dbh_min, mass_min, area) {
  si <- ifelse(dbh1 >= dbh_min & dbh2 >= dbh_min, 1, 0) # survival
  di <- ifelse(dbh1 >= dbh_min & dbh2 < dbh_min, 1, 0) # death
  ri <- ifelse(dbh1 < dbh_min & dbh2 >= dbh_min, 1, 0) # recruitment

  Ns0 <- sum(si, na.rm = TRUE)
  N0 <- Ns0 + sum(di, na.rm = TRUE)
  NT <- Ns0 + sum(ri, na.rm = TRUE)
  Bs0 <- sum(si * w1, na.rm = TRUE)
  BsT <- sum(si * w2, na.rm = TRUE)
  B0 <- Bs0 + sum(di * w1, na.rm = TRUE)
  BT <- BsT + sum(ri * w2, na.rm = TRUE)
  # period-mean biomass and abundance
  Nw <- ifelse(NT != N0, (NT - N0) / log(NT / N0), N0)
  N <- Nw / area # (per ha)
  Bw <- ifelse(BT != B0, (BT - B0) / log(BT / B0), B0)
  B <- Bw / area

  # Standardized maximum tree mass for initial population
  W_max <- as.numeric(quantile(w1[ri != 1], 0.99)) # Mg

  # turnover rates
  r <- turnover(si + ri, si, t)
  m <- turnover(si + di, si, t)
  p <- turnover(w2, si * w1, t)
  l <- turnover(w1, si * w1, t)

  # absolute productivity (Mg per ha per year)
  P <- p * B
  P_simple <- sum(((si + ri) * w2 - si * w1) / t)
  P_simple_Clark <- sum(si * (w2 - w1) / t + ri * (w2 - mass_min) / t)
  P_simple <- P_simple / area
  P_simple_Clark <- P_simple_Clark / area # cf. Clark et al. (2001, Ecol. Appl.)

  return(list(
    "B" = B,
    "N" = N,
    "W_max" = W_max,
    "p" = p,
    "l" = l,
    "r" = r,
    "m" = m,
    "P" = P,
    "P_simple" = P_simple,
    "P_simple_Clark" = P_simple_Clark
  ))
}

# Load data ---------------------------
df1 <- read.csv("data/pfr_observed.csv.gz", header = TRUE)
df2 <- read.csv("data/pfr_identity_free.csv.gz", header = TRUE)

# Boundary size in dbh (cm)
dbh_min <- 1.

data_preparation <- function(d) {
  # remove rows if both dbh1 and dbh2 lower than dbh_min
  d <- filter(d, dbh1 >= dbh_min | dbh2 >= dbh_min)

  # add some columns to the dataframe
  d <- d %>%
    mutate(
      dbh1 = ifelse(dbh1 >= dbh_min, dbh1, 0),
      dbh2 = ifelse(dbh2 >= dbh_min, dbh2, 0),
      w1 = biomass(dbh1) / 1000, # tree biomass in Mg
      w2 = biomass(dbh2) / 1000,
    )

  # Select species with NsT >= 100
  counts <- table(pull(filter(d, dbh1 >= dbh_min & dbh2 >= dbh_min), Cd))
  sp_list <- names(counts)[counts >= 100]

  # rename the remaining species to 'Others'
  d <- d %>%
    mutate(
      Cd = ifelse(Cd %in% sp_list, as.character(Cd), "Others"),
      Cd = factor(Cd, levels = c(sp_list, "Others"))
    )

  # sort Cd to make 'Others' to be the last
  sp_order <- order(d$Cd)
  d <- d[sp_order, ]
  return(d)
}

df1 <- data_preparation(df1)
df2 <- data_preparation(df2)

# Estimate production rate ------------
# for each Cd
dbh_min <- 1.
mass_min <- biomass(2) / 1000 # tree biomass in Mg at dbh = 2 cm
area <- 50 # total area in ha

res1 <- df1 %>%
  group_nest(Cd) %>%
  mutate(y = purrr::map(data, ~ as_tibble(
    with(., productivity(dbh1, dbh2, w1, w2, t, dbh_min, mass_min, area))
  ))) %>%
  select(-data) %>%
  unnest(cols = y)

res2 <- df2 %>%
  group_nest(Cd) %>%
  mutate(y = purrr::map(data, ~ as_tibble(
    with(., productivity(dbh1, dbh2, w1, w2, t, dbh_min, mass_min, area))
  ))) %>%
  select(-data) %>%
  unnest(cols = y)

# write results
outdir <- "output"
dir.create(outdir, showWarnings = FALSE)
write.csv(res1, file.path(outdir, "res_pfr_observed.csv"), row.names = FALSE)
write.csv(res2, file.path(outdir, "res_pfr_identity_free.csv"), row.names = FALSE)

# Plot productivity ~ biomass ---------
# Cd-sum plot-level productivity (Mg per ha per year)
Ps1 <- with(res1, c(sum(p * B), sum(P_simple), sum(P_simple_Clark)))
Ps2 <- with(res2, c(sum(p * B), sum(P_simple), sum(P_simple_Clark)))
names(Ps1) <- c("P", "P_simple", "P_simple_Clark")
names(Ps2) <- c("P", "P_simple", "P_simple_Clark")
print("Observed")
print(Ps1)
print("Identity free")
print(Ps2)

# fit the linear regression model
fit1 <- lm(log(p) ~ log(B), data = filter(res1, Cd != "Others"))
pval1 <- summary(fit1)$coefficients[2, 4]
print("Observed")
print(summary(fit1))

fit2 <- lm(log(p) ~ log(B), data = filter(res2, Cd != "Others"))
pval2 <- summary(fit2)$coefficients[2, 4]
print("Identity-free")
print(summary(fit2))

# draw a plot
xbreaks = c(.01, 1, 100)
p1 <- ggplot(filter(res1, Cd != "Others"), aes(x = B, y = p)) +
  geom_point(alpha = .5) +
  geom_point(data = filter(res1, Cd == "Others"), shape = 18, size = 3,  alpha = .5) +
  annotate(
    "curve",
    x = 20, y = .12, xend = 45.4, yend = .028,
    curvature = -.25,
    arrow = arrow(length = unit(2, "mm"))
  ) +
  annotate("text", x = 10, y = .2, label = "rare species\ncombined", hjust = .5) +
  scale_x_log10(limits = c(.0005, 100), breaks = xbreaks, labels = xbreaks) +
  scale_y_log10(limits = c(.001, .5)) +
  labs(
    title = "(a) observed",
    x = parse(text = "Biomass*','~B~(Mg %.% ha^{-1})"),
    y = parse(text = "Relative~productivity*','~p~(year^{-1})")
  )

p2 <- ggplot(filter(res2, Cd != "Others"), aes(x = B, y = p)) +
  geom_point(alpha = .5) +
  geom_point(data = filter(res2, Cd == "Others"), shape = 18, size = 3,  alpha = .5) +
  scale_x_log10(limits = c(.0005, 100), breaks = xbreaks, labels = xbreaks) +
  scale_y_log10(limits = c(.001, .5)) +
  labs(
    title = "(b) identity-free",
    x = parse(text = "Biomass*','~B~(Mg %.% ha^{-1})"),
    y = parse(text = "Relative~productivity*','~p~(year^{-1})")
  )

if (pval1 < .05) {
  p1 <- p1 + geom_smooth(method = "lm", formula = y ~ x)
}

if (pval2 < .05) {
  p2 <- p2 + geom_smooth(method = "lm", formula = y ~ x)
}

p <- cowplot::plot_grid(p1, p2)

# save a plot
outdir <- "figs"
dir.create(outdir, showWarnings = FALSE)
ggsave(p, file = file.path(outdir, "p-B.png"), width = 8.5, height = 4)


# Inverse function
myginv <- function (X, tol = sqrt(.Machine$double.eps)){
    ## the ginv function for symmetric matrices. it avoid some errors.
        if (length(dim(X)) > 2L || !(is.numeric(X) || is.complex(X)))
                  stop("'X' must be a numeric or complex matrix")
            if (!is.matrix(X))
                      X <- as.matrix(X)
	    if(!isSymmetric(X))
		      X <- (X + t(X))/2
            Xeigen <- eigen(X,symmetric=T)
            Positive <- Xeigen$values > max(tol * Xeigen$values[1L], 0)
            if (all(Positive))
                      Xeigen$vectors %*% (1/Xeigen$values * t(Xeigen$vectors))
            else if (!any(Positive))
                      array(0, dim(X)[2L:1L])
            else Xeigen$vectors[, Positive, drop = FALSE] %*% ((1/Xeigen$values[Positive]) *
                                              t(Xeigen$vectors[, Positive, drop = FALSE]))
}

mtxInvPrx <- function(d,w,LD.matrix,thrw,thrctd=1,thrp=length(d)/2,thrp2=thrp/2,thrp3=5){ ### This function approximates the inverse of D+diag(w)%*%LD.matrix%*%diag(w)
 p <- length(w)
 wsq <- w^2
 rkwsq <- rank(wsq)
 wsqcumsum <- sapply(1:p, function(ii) sum(wsq[rkwsq<=ii]))
 wstat <- wsqcumsum+sqrt((p-(1:p))*wsqcumsum)
 rkthr <- min(max(which(wstat<thrw),p-thrp),p-thrp3) ## at least throw out half of the variables
 ix1 <- (rkwsq>rkthr)
 if(sum(ix1) != 1){
   ctdmax <- apply(abs(LD.matrix-diag(diag(LD.matrix)))[ix1,],2,max)
   ctdmax[ix1] <- 0
   ix1 <- ix1|(ctdmax>=thrctd)
   ix2 <- !ix1

   M0 <- sapply(1:p, function(ii) w[ii]*LD.matrix[ii,])
   M0 <- sapply(1:p, function(ii) w[ii]*M0[ii,])
   M <- diag(d)+M0
   Minv <- diag(1/d)
   if(isSymmetric(M[ix1,ix1])){
     Minv[ix1,ix1] <- myginv(M[ix1,ix1])
   }else Minv[ix1,ix1] <- myginv((M[ix1,ix1]+t(M[ix1,ix1]))/2)#myginv(M[ix1,ix1])
   out <- Minv
 }else out <- 'ginv'
 out
}

vestepDirectSS <- function(posterior, para, LD.matrix, wzy, ycy, anno = matrix(1, length(wzy), 1), thrw.mtxinv = 0.1, thrctd.mtxinv = 0.8, thrp.mtxinv = length(wzy)){
	p <- length(wzy)
	gammabeta <- para$gammabeta
	lodanno <- anno%*%gammabeta
	qlodbeta <- matrix(posterior$lodbeta,p,1)
	qbeta <- 1/(1+exp(-qlodbeta))

	### update the posterior distribution of the prior distribution of pibeta
	updatePibeta <- function(expanno, qbeta){
		shape1pibeta <- expanno + qbeta
		shape2pibeta <- 1/expanno+1-qbeta
		epibeta <- shape1pibeta/(shape1pibeta+shape2pibeta)
		elogpibeta <- digamma(shape1pibeta) - digamma(shape1pibeta+shape2pibeta)
		elogpibetac <- digamma(shape2pibeta) - digamma(shape1pibeta+shape2pibeta)
		elodpibeta <- elogpibeta - elogpibetac
		out <- list(par = list(shape1pibeta = shape1pibeta, shape2pibeta = shape2pibeta), stat = list(epibeta = epibeta,
			elodpibeta = elodpibeta, elogpibeta = elogpibeta, elogpibetac = elogpibetac))
		out
	}

	### update the posterior distribution of tau
	updateTau <- function(para, qbeta){
		vareps <- para$vareps
		nutau <- para$nutau
		At <- try(myginv(diag(as.vector(qbeta))%*%LD.matrix%*%diag(as.vector(qbeta))+diag(as.vector(qbeta-qbeta^2))+diag(1,p)/nutau))
		if (class(At) == 'try-error') {
			cat('Caught an error during full inversion, trying approximation.\n')
			At <- mtxInvPrx(d=1/nutau+as.vector(qbeta-qbeta^2), w=qbeta, LD.matrix=LD.matrix, thrw=thrw.mtxinv, thrctd=thrctd.mtxinv, thrp=thrp.mtxinv, thrp2=thrp.mtxinv/2)
		}
		vartau <- vareps*At
		mutau <- At%*%(qbeta*wzy)
		out <- list(A = At, vartau = vartau, mutau = mutau)
		out
	}

	post.tau <- updateTau(para,qbeta)

	### update the posterior distribution of sbeta (signal of each SNP)
	updateSbeta <- function(para, lodprior, posttau, qbeta){
		vareps <- para$vareps
		vartau <- posttau$vartau
		mutau <- posttau$mutau
		muwzy <- LD.matrix%*%(mutau*qbeta)
		res <- (wzy - muwzy)
		post.lodbeta <- lodprior-(1/(2*vareps))*(diag(LD.matrix)*(mutau^2+diag(vartau))-2*mutau*res-2*mutau^2*qbeta+2*(LD.matrix*vartau)%*%qbeta-2*diag(LD.matrix)*diag(vartau)*qbeta)
		out <- list(lodbeta = post.lodbeta)
		out
	}

	post.sbeta <- updateSbeta(para, lodanno, posttau = post.tau, qbeta)

	out <- list(posterior = c(post.sbeta, post.tau), stat = NULL)
	out
} ### End of "vestepDirectSS"

vmstepDirectSS <- function(post, LD.matrix, wzy, ycy, init = list(gammabeta = rep(0, dim(anno)[2]), vareps = 1, nutau = 5), anno = matrix(1, length(wzy), 1), iter.max.mstep = 100){
	p <- length(wzy)
	mutau <- post[['posterior']]$mutau
	vartau <- post[['posterior']]$vartau
	lodbeta <- post[['posterior']]$lodbeta
	qbeta <- 1/(1+exp(-lodbeta))
	mubeta <- mutau*qbeta
	qdly <- (ycy-2*sum(wzy*mubeta)+sum(mubeta*(LD.matrix%*%mubeta))+sum(qbeta*((LD.matrix*vartau)%*%qbeta))+sum(diag(LD.matrix)*(qbeta-qbeta^2)*(mutau^2+diag(vartau))))
	qdltau <- sum((mutau^2+diag(vartau))*qbeta)

	vareps <- qdly/(p-sum(qbeta))
	nutau <- qdltau/sum(qbeta)/vareps

	loglkpibetaprior <- function(gammabeta){
		lodanno <- anno%*%matrix(gammabeta, ncol = 1)
		out <- -qbeta*lodanno+log(1+exp(lodanno))
		sum(out)
	}

	out.loglkpibetaprior <- optim(par = init$gammabeta, fn = loglkpibetaprior, method = "BFGS", hessian = TRUE, control = list(maxit = iter.max.mstep))
	out <- list(par = list(gammabeta = out.loglkpibetaprior$par, vareps = vareps, nutau = nutau), hess = list(pibetaprior = out.loglkpibetaprior$hessian))
	out
} ### End of "vmstepDirectSS"

vemDirectSS <- function(LD.matrix, GEMsummstats, anno = NULL, iter.max = 200, iter.max.mstep = 100, er.max = 1e-5,
          init = list(gammabeta = c(-1, rep(0, dim(anno)[2] - 1)), vareps = sd(GEMsummstats)^2, nutau = 2),
          posterior.init = list(lodbeta = matrix(0, length(GEMsummstats), 1))){
  thrw.mtxinv = 0.1
  thrctd.mtxinv=0.8
  wzy <- GEMsummstats
	p <- length(wzy)
	if(is.null(anno)) anno <- matrix(1, p, 1)
	LD.matrix.inv <- myginv(LD.matrix)
	ycy <- as.numeric(t(wzy)%*%LD.matrix.inv%*%wzy)
	posterior.old <- posterior.init
	wzy <- matrix(wzy,p,1)

	vaicDirect <- function(para,post){
 		mutau <- post[['posterior']]$mutau
		vartau <- post[['posterior']]$vartau
		lodbeta <- post[['posterior']]$lodbeta
		vareps <- para[['vareps']]
		qbeta <- 1/(1+exp(-lodbeta))
		mubeta <- mutau*qbeta
		p <- length(lodbeta)
		qdly <- (ycy-2*sum(wzy*mubeta)+sum(mubeta*(LD.matrix%*%mubeta))+sum(qbeta*((LD.matrix*vartau)%*%qbeta))+sum(diag(LD.matrix)*(qbeta-qbeta^2)*(mutau^2+diag(vartau))))
		ely <- -qdly/(2*vareps) -(p/2)*log(vareps)
		mly <- -(ycy-2*sum(wzy*mubeta)+sum(mubeta*(LD.matrix%*%mubeta)))/(2*vareps) -(p/2)*log(vareps)
		pd <- 2*(mly-ely)
		vaic <- -2*mly+2*pd
		vaic
	}

	er <- Inf
	ll.all <- Inf
	ll.old <- Inf
	iter <- 1
	mstep.all <- list()

	### The algorithm stop until the hyperparameter estimation converges in relative error.
	while((abs(er)>er.max)&(iter<iter.max)){
		estep <- vestepDirectSS(posterior.old, init, LD.matrix, wzy, ycy, anno, thrw.mtxinv = thrw.mtxinv, thrctd.mtxinv = thrctd.mtxinv, thrp.mtxinv = floor(p*(1/4+(3/4)^iter)))
		mstep <- vmstepDirectSS(post = estep, LD.matrix, wzy, ycy, init = init, anno, iter.max.mstep = iter.max.mstep)
		mstep.all[[iter]] <- mstep
		posterior.old <- estep[['posterior']]
		init <- mstep$par
		ll <- vaicDirect(init, estep)
		ll.all <- c(ll.all, ll)
		er <- ll.old - ll
		er <- max(er,er/abs(ll))
		ll.old <- ll
		iter <- iter + 1
	}

	out <- list(posterior = estep[['posterior']], par = mstep$par, niter = iter,
		converged = (er<er.max), par.all = mstep.all)
	out
} ### End of "vemDirectSS"

vestepMedSS <- function(posterior,para,LD.matrix,wzy,ycy,wzr,rcr,rcy,anno=matrix(1,length(wzy),1),thrw.mtxinv=0.1,thrctd.mtxinv=0.8,thrp.mtxinv=length(wzy)){
  p <- length(wzy)
  gammabeta <- para$gammabeta
  lodanno <- anno%*%gammabeta
  qlodbeta <- matrix(posterior$lodbeta,p,1)
  qbeta <- 1/(1+exp(-qlodbeta))
  m1gamma <- posterior$mudelta

  ### update the posterior distribution of the prior distribution of pibeta
  updatePibeta <- function(expanno, qbeta){
    shape1pibeta <- expanno+qbeta
    shape2pibeta <- 1/expanno+1-qbeta
    epibeta <- shape1pibeta/(shape1pibeta+shape2pibeta)
    elogpibeta <- digamma(shape1pibeta)-digamma(shape1pibeta+shape2pibeta)
    elogpibetac <- digamma(shape2pibeta)-digamma(shape1pibeta+shape2pibeta)
    elodpibeta <- elogpibeta-elogpibetac
    out <- list(par=list(shape1pibeta=shape1pibeta,shape2pibeta=shape2pibeta),stat=list(epibeta=epibeta,elodpibeta=elodpibeta,elogpibeta=elogpibeta,elogpibetac=elogpibetac))
    out
  }

  ### update the posterior distribution of tau
  updateTau <- function(para,qbeta){
    vareps <- para$vareps
    nutau <- para$nutau
    At <- try(myginv(diag(as.vector(qbeta))%*%LD.matrix%*%diag(as.vector(qbeta))+diag(as.vector(qbeta-qbeta^2))+diag(1,p)/nutau))
    if (class(At) == 'try-error') {
      cat('Caught an error during full inversion, trying approximation.\n')
      At <- mtxInvPrx(d=1/nutau+as.vector(qbeta-qbeta^2), w=qbeta, LD.matrix=LD.matrix, thrw=thrw.mtxinv, thrctd=thrctd.mtxinv, thrp=thrp.mtxinv, thrp2=thrp.mtxinv/2)
    }
    vartau <- vareps*At
    mutau <- At%*%(qbeta*(wzy-m1gamma*wzr))
    out <- list(A=At,vartau=vartau,mutau=mutau)
    out
  }

  post.tau <- updateTau(para,qbeta)

  updateDelta <- function(para,posttau,qbeta){
    vareps <- para$vareps
    mutau <- posttau$mutau
    mudelta <- (rcy-sum(wzr*mutau*qbeta))/rcr
    vardelta <- vareps/rcr
    out <- list(mudelta=mudelta,vardelta=vardelta)
    out
  }

 post.delta <- updateDelta(para,post.tau,qbeta)

  ### update the posterior distribution of sbeta
  updateSbeta <- function(para,lodprior,posttau,postdelta,qbeta){
    vareps <- para$vareps
    vartau <- posttau$vartau
    mutau <- posttau$mutau
    mudelta <- postdelta$mudelta
    muwzy <- LD.matrix%*%(mutau*qbeta)+mudelta*wzr
    res <- (wzy -muwzy)
    post.lodbeta <- lodprior-(1/(2*vareps))*(diag(LD.matrix)*(mutau^2+diag(vartau))-2*mutau*res-2*mutau^2*qbeta+2*(LD.matrix*vartau)%*%qbeta-2*diag(LD.matrix)*diag(vartau)*qbeta)
    out <- list(lodbeta=post.lodbeta)
    out
  }

  post.sbeta <- updateSbeta(para,lodanno,posttau=post.tau,postdelta=post.delta,qbeta)

  out <- list(posterior=c(post.sbeta,post.tau,post.delta),stat=NULL)
  out
} ### End of "vestepMedSS"

vmstepMedSS <- function(post, LD.matrix, wzy, ycy, wzr, rcr, rcy, init = list(gammabeta = rep(0, dim(anno)[2]), vareps = 1, nutau = 5),
				anno = matrix(1, length(wzy), 1), iter.max.mstep = 100){

	p <- length(wzy)
	mutau <- post[['posterior']]$mutau
	vartau <- post[['posterior']]$vartau
	mudelta <- post[['posterior']]$mudelta
	vardelta <- post[['posterior']]$vardelta
	lodbeta <- post[['posterior']]$lodbeta
	qbeta <- 1/(1+exp(-lodbeta))
	mubeta <- mutau*qbeta
	qdly <- (ycy-2*sum(wzy*mubeta)+sum(mubeta*(LD.matrix%*%mubeta))+sum(qbeta*((LD.matrix*vartau)%*%qbeta))+sum(diag(LD.matrix)*(qbeta-qbeta^2)*(mutau^2+diag(vartau)))+(mudelta^2+vardelta)*rcr-2*mudelta*(rcy-sum(wzr*mubeta)))

	qdltau <- sum((mutau^2+diag(vartau))*qbeta)
	vareps <- qdly/(p-sum(qbeta)-1)
	nutau <- qdltau/sum(qbeta)/vareps

	loglkpibetaprior <- function(gammabeta){
		lodanno <- anno%*%matrix(gammabeta, ncol = 1)
		out <- log(1 + exp(lodanno)) - qbeta*lodanno
		sum(out)
	}
	out.loglkpibetaprior <- optim(par = init$gammabeta, fn = loglkpibetaprior, method = "BFGS", hessian = TRUE, control = list(maxit = iter.max.mstep))
	out <- list(par = list(gammabeta = out.loglkpibetaprior$par, vareps = vareps, nutau = nutau), hess = list(pibetaprior = out.loglkpibetaprior$hessian))
	out
} ### End of "vmstepMedSS"

vemMedSS <- function(LD.matrix, DEMsummstats, GEMsummstats, anno = NULL, iter.max = 200, iter.max.mstep = 100, er.max = 1e-5,
	init = list(gammabeta = c(-1, rep(0, dim(anno)[2] - 1)), vareps = sd(DEMsummstats)^2, nutau = 2),
	posterior.init = list(lodbeta = matrix(0, length(DEMsummstats), 1), mudelta = 0)){
  thrw.mtxinv = 0.1
  thrctd.mtxinv = 0.8
  wzy <- DEMsummstats
  wzr <- GEMsummstats
	p <- length(wzy)
	if(is.null(anno)) anno <- matrix(1, p, 1)
	LD.matrix.inv <- myginv(LD.matrix)
  ycy <- as.numeric(t(wzy)%*%LD.matrix.inv%*%wzy)
  rcr <- as.numeric(t(wzr)%*%LD.matrix.inv%*%wzr)
  rcy <- as.numeric(t(wzr)%*%LD.matrix.inv%*%wzy)

	posterior.old <- posterior.init
	wzy <- matrix(wzy,p,1)

	vaicMed <- function(para,post){
		mutau <- post[['posterior']]$mutau
		vartau <- post[['posterior']]$vartau
		mudelta <- post[['posterior']]$mudelta
		vardelta <- post[['posterior']]$vardelta
		lodbeta <- post[['posterior']]$lodbeta
		vareps <- para[['vareps']]
		qbeta <- 1/(1+exp(-lodbeta))
		mubeta <- mutau*qbeta
		p <- length(lodbeta)

		qdly <- (ycy-2*sum(wzy*mubeta)+sum(mubeta*(LD.matrix%*%mubeta))+sum(qbeta*((LD.matrix*vartau)%*%qbeta))+sum(diag(LD.matrix)*(qbeta-qbeta^2)*(mutau^2+diag(vartau)))+(mudelta^2+vardelta)*rcr-2*mudelta*(rcy-sum(wzr*mubeta)))

		ely <- -qdly/(2*vareps) -(p/2)*log(vareps)
		mly <- -(ycy-2*sum(wzy*mubeta)+sum(mubeta*(LD.matrix%*%mubeta))+mudelta^2*rcr-2*mudelta*(rcy-sum(wzr*mubeta)))/(2*vareps) -(p/2)*log(vareps)
		pd <- 2*(mly-ely)
		vaic <- -2*mly+2*pd
		vaic
	}

	er <- Inf
	ll.all <- Inf
	ll.old <- Inf
	iter <- 1
	mstep.all <- list()
	### The algorithm stop until the hyperparameter estimation converges in relative error.
	while((abs(er)>er.max)&(iter<iter.max)){
		estep <- vestepMedSS(posterior.old, init, LD.matrix, wzy, ycy, wzr, rcr, rcy, anno, thrw.mtxinv = thrw.mtxinv, thrctd.mtxinv = thrctd.mtxinv,
			thrp.mtxinv = floor(p*(1/4+(3/4)^iter)))
		mstep <- vmstepMedSS(post = estep,LD.matrix, wzy, ycy, wzr, rcr, rcy, init = init, anno, iter.max.mstep = iter.max.mstep)
		mstep.all[[iter]] <- mstep
		posterior.old <- estep[['posterior']]
		init <- mstep$par
		ll <- vaicMed(init, estep)
		ll.all <- c(ll.all, ll)
		er <- ll.old - ll
		er <- max(er,er/abs(ll))
		ll.old <- ll
		iter <- iter + 1
  }
	out <- list(posterior = estep[['posterior']], par = mstep$par, niter = iter, converged = (er<er.max),
		par.all = mstep.all)
	out
} ### End of "vemMedSS"

getParameters <- function(omod){  ### process the output of the model without mediator
	omod[['par']]
}

getPostProb <- function(omod){
	PP <- 1/(1+exp(- omod[['posterior']]$lodbeta))
	SNP.name <- rownames(PP)
	PP <- c(PP)
	names(PP) <- SNP.name
	idfdr <- rank(-PP, ties.method = 'max')
	fdr <- sapply(idfdr, function(ii) 1 - mean(PP[idfdr <= ii]))

  out <- PP
	out
}

processFunMed <- function(GEMoutput, DEMoutput){

  mod <- list()
  mod[['model.opt']] <- list(GEM = GEMoutput, DEM = DEMoutput)

	mod.out <- list()
	mod.out[['Parameters']] <- lapply(mod[['model.opt']], getParameters)
	names(mod.out[['Parameters']][['GEM']]) <- c('gammaB', 'varEta', 'nuB')
	names(mod.out[['Parameters']][['DEM']]) <- c('gammaBeta', 'varEpsilon', 'nuBeta')
	mod.out[['Parameters']][['DEM']][['delta']] <- mod[['model.opt']][['DEM']][['posterior']]$mudelta
	mod.out[['PostProb']] <- lapply(mod[['model.opt']], getPostProb)
	mod.out[['Convergence']] <- lapply(mod[['model.opt']], function(xx) xx[c('niter','converged')])

	mod.out
}

meanAnno <- function(x,y){
  y <- (as.numeric(y)>0)
  if(sum(y)>0){ x1 <- mean(x[y])
  }else{ x1 <- 0}
  x1
}

lbAnno <- function(x, y, post, nrep=1000){
  y <- as.numeric(y)
  n <- length(y)
  n1 <- sum(y>0)
  shm <- sapply(1:nrep, function(ii){
    idii <- sample(n,n1,replace=FALSE)
    out <- mean(x[idii])
    out })
  out <-  sum(shm >= post)/nrep
  out
}

annoEnrich <- function(iFunMedNull, annoMtx, Nperm = 5000, cores = 20){
  require(parallel)
  idanno <- colnames(annoMtx)
  ## posterior porbabilities from the null model
  post.null <- list(GEM = iFunMedNull[['PostProb']][['GEM']],
                    DEM = iFunMedNull[['PostProb']][['DEM']])

  ## average posterior probability of inclusion of the SNPs with the annotation, for each annotation
  post.meananno.null <- lapply(post.null, function(x) apply(annoMtx, 2, meanAnno, x = x))

  anno.enrichment <- mclapply(names(post.null), function(x) sapply(1:length(idanno), function(y) lbAnno(post.null[[x]], annotation.matrix[, y], post.meananno.null[[x]][y], nrep = Nperm)), mc.cores = cores, mc.preschedule = FALSE)
  names(anno.enrichment) <- names(post.null)
  names(anno.enrichment[['GEM']]) <- names(post.meananno.null[['GEM']])
  names(anno.enrichment[['DEM']]) <- names(post.meananno.null[['DEM']])

  return(list(enrichmentPval = anno.enrichment, avePP = post.meananno.null))
}


#' Effective number of codons
#'
#' This function calculate the effective number of codons (ENC) using Wright's formula.
#'
#' @param x a list of KZsqns objects
#' @param spp.list an array indicating the the species identity of each KZsqns object in the list \emph{x}. The default option is to calculate an ENC for each KZsqns object. When \emph{spp.list} is specified, an ENC is calculated for each unique element of \emph{spp.list}
#' @param y a number to assign the type of codon table to be used for the translation. 0 = standard codon table, 2 = vertibrate mitchodrial (See dna2aa for additional options). If no number is specified or an option other than those provided above is specified, there will be a warning and the standard codon table will be used
#' @param formula a char array to denote the formula for calculating ENC. Options include: 'w': Wright's formula, 'f4-': Fuglsang's Ncf4- formula.
#' @return an numeric array of the ENCs
#' @section Warning:
#' According to Wright's formula, the ENC is the sum of the heterogeneity of all the degeneracy classes. When all the codons belonging to a degeneracy class is missing, the ENC can not be calculated. A warning will result when it happens. The most likely situation leading to such a condition is that the sequence of codons is too short. When the sequence is short, the calculated ENC is likely to deviate much from the underlying value. It is recommended to increase the sample size to solve this issue when Wright's formula is prefered. Alternatively, Fuglsang's Ncf4- formula could be used instead.
#' @references
#' Fuglsang, A., 2006 Estimating the "Effective Number of Codons": The Wright way of determining codon homozygosity leads to superior estimates. Genetics 172:1301-1307
#' Wright, F., 1990 The 'effective number of codons' used in a gene. Gene 87: 23-29.
#' @examples
#' dnalist = vector('list', 2)
#' dnalist[[1]] = CodonTable2[sample(1:64, 1e4, TRUE),1]
#' attr(dnalist[[1]], 'class') <- 'KZsqns'
#' dnalist[[2]] = CodonTable2[sample(1:64, 1e4, TRUE),1]
#' attr(dnalist[[2]], 'class') <- 'KZsqns'
#' enc(x = dnalist, y = 2) # Two numbers should be around 60 as there are 60 animo acid coding codons and 4 stop codons for codon table 2
#'
#' dnalist = vector('list', 2)
#' dnalist[[1]] = CodonTable2[sample(1:64, 1e4, TRUE),1]
#' attr(dnalist[[1]], 'class') <- 'KZsqns'
#' dnalist[[2]] = CodonTable2[sample(1:64, 1e4, TRUE),1]
#' attr(dnalist[[2]], 'class') <- 'KZsqns'
#' enc(x = dnalist, y = 4) # Two numbers should be around 62 as there are 60 animo acid coding codons and 2 stop codons for codon table 4
#' @export

enc <- function(x, y=0, spp.list='n', formula = 'w'){
  if(!is.list(x)) x = list(x)
  cTable = arg_check(x[[1]], "KZsqns", y)
  calc = nc_wright
  switch(formula,
         'f4-' = {calc = nc_fuglsang4m},
         'w'   = {},
         {
           warning("Not a valid formula, and Wright's formula will be used.")
         })
  if(length(spp.list)==1 && spp.list=='n'){
    cat("No species list is provided\nCalculating ENCs for each KZsqns object.\n")
    return(sapply(x, function(i){calc(table(i), cTable)}))
  } else {
    if (length(spp.list)!=length(x)) stop("The lengths of the species list and the list of KZsqns do not match")
    spp = unique(spp.list)
    fixx = function(i){
      jobs = which(spp.list==spp[i])
      ans = rep(0, 64)
      for(j in jobs){
        ans = rbind(ans, table(x[[j]])[cTable[,1]])
      }
      ans = colSums(ans, na.rm = TRUE)
      names(ans) <- cTable[,1]
      return(ans)
    }
    ans = sapply(1:length(spp), function(i){calc(fixx(i), cTable)})
    names(ans) <- spp
    return(ans)
  }
}


# rocauc.r - plot ROC and compute AUC given score+label table with R using RROC
#
# Alex Stivala, October 2008
#
# Requires the ROCR package from CRAN (developed with version 1.0-2)
# (ROCR in turn requires gplots, gtools, gdata)
#
# Run this on the output of e.g. tsevalfn.py with the -l option,
# it is a table with one column of scores from classifier, and second
# column of true class label (0 or 1)
#
# Uses commandArgs() R function to get trailing arguments from R command
# line ie after the --args option. The filename of the .slrtab file
# is obtained from --args, and the output file is constructed form it
# eg foo.slrtab results in foo.eps file for ROC plot with
#
#    R --vanilla -f rocauc.r --args foo.slrtab
#
#
# The citation for the ROCR package is
#   Sing et al 2005 "ROCR: visualizing classifier performance in R"
#   Bioinformatics 21(20):3940-3941
#
#
# Also calculate stdand eror (and 95% confidence inteval) for the AUC,
# according to Hanley-McNeil method:
#   Hanley & McNeil 1982 "The Meaning and Use of the Area under a Receiver
#   Operating Characteristic (ROC) Curve" Radiology 143(1):29-36
# ROCR has all sorts of featuers but this isn't one of them so had
# to implement it here.
#

# $Id: rocauc.r 3606 2010-05-04 06:03:55Z alexs $

library(ROCR)

#
# functions
#

#
# compute AUC and standard error and 95% CI by Hanley-McNeil method,
# using R wilcox.test for the Wilcoxon rank-sum (aka Mann-Whitney) test
#
# Parameters:
#   tab : data frame with score and label columns
#
# Return value:
#   list with auc and stderror members
#
compute_auc_error <- function(tab)
{
    # tab is a data frame with score and label columns
    x <- tab$score[tab$label == 1]
    y <- tab$score[tab$label == 0]
    wilcox <- wilcox.test(x,y)
    nA <- length(x)
    nN <- length(y)
    stopifnot(nA + nN == length(tab$label))
    nA <- as.double(nA)
    nN <- as.double(nN)
    U <- wilcox$statistic  # Mann-Whitney U statistic
    theta <- U / (nA * nN) # AUC
    theta2 <- theta*theta
    Q1 <- theta / (2 - theta)
    Q2 <- 2*theta2 / (1 + theta)
    SE2 <- (theta*(1-theta) + (nA - 1)*(Q1 - theta2) + (nN - 1)*(Q2 - theta2)) /
          (nA*nN)
    SE <- sqrt(SE2)

    retval <- list()
    retval$auc <- theta
    retval$stderror <- SE
    return(retval)
}

#
# plot the ROC curve and compute AUC using ROCR
#
plotroc <- function(filename)
{
    tab <- read.table(filename, header=TRUE)
    # tab is a data frame with score and label columns
    pred <- prediction(tab$score, tab$label)
    perfroc <- performance(pred, measure="tpr",x.measure="fpr")
    perfauc <- performance(pred, measure="auc")

    # EPS suitable for inserting into LaTeX
    postscript(sub('[.]slrtab$','.eps',filename),
               onefile=FALSE,paper="special",horizontal=FALSE,
               width = 9, height = 6)
    plot(perfroc)
    auc <- perfauc@y.values
    legend('bottomright', legend=paste('AUC =', format(auc, digits=4)), bty='n')
    dev.off()

    aucerr <- compute_auc_error(tab)
    quantile <- 1.96 # 0.975 quantile of normal distrib
    low95 <- aucerr$auc - quantile * aucerr$stderr
    high95 <- aucerr$auc + quantile * aucerr$stderr

    cat(filename,':\n')
    cat('RROC          AUC =',format(perfauc@y.values,digits=4),'\n')
    cat('Hanley-McNeil AUC =',format(aucerr$auc,digits=4),'\n')
    cat('       std. error =',format(aucerr$stderror,digits=5),'\n')
    cat('           95% CI =',format(low95,digits=4),',',
                               format(high95,digits=4),'\n')
}

#
# main
#
filename <- commandArgs(trailingOnly=TRUE)
plotroc(filename)


#' Time derivative of DO concentration
#'
#' @param t Time step for the ode integration
#' @param y Current state (dissolved oxygen concentration)
#' @param parms a named vector of parameters; see details
#' @param data a list of data for the ode; see details
#' @param return_list logical, should a list (suitable for deSolve) be returned?
#' @details `params` must be a vector including the following named items:
#' * `P1` $(W min g^{-1} O_2)$ inverse of the slope of a photosynthesis–irradiance curve at low
#' 		 light intensity
#' * `P2` $(m^2 min g^{-1} O_2)$ inverse maximum photosynthesis rate; can be zero to assume
#' 		  GPP is linear with light intensity instead of saturating
#' * `k600` coefficient of gas exchange for a gas with a Schmidt number of 600 (see [kT()]).
#' * `ER24_20` daily ecosystem respiration rate, standardized at 20 degrees C
#'
#' `data` is a named list of (constant) data items, including the following:
#' * `PAR`  a function that takes time as a parameter and returns light intensity $(W/m^2)$
#' * `temp` a function that takes time as a parameter and returns water temperature (degrees C)
#' * `P`    pressure, in atmospheres
#' * `z`    Depth, in meters
#'
#' @references Fuß, T. et al. (2017). Land use controls stream ecosystem metabolism by shifting
#' 		 dissolved organic matter and nutrient regimes. *Freshw Biol* **62**:582–599.
#'
#'		 Van de Bogert MC et al. 2007. Assessing pelagic and benthic metabolism using free
#' 		 water measurements. *Limnol. Oceanogr. Methods* **5**:145–155.
#'
#' @return Time derivative of dissolved oxygen
#' @export
oneStation_dDOdt <- function(t, y, parms, data, return_list=FALSE)
{
	warning("oneStation_dDOdt is deprecated; use DOCalibration(..., method = 'stan' instead)")
	# gross primary production; Fuß et al eq 2
	GPP <- gpp(data$PAR(t), parms['P1'], parms['P2'])

	# rearation flux, Van de Bogert et al eqn 4; note that k600 is fixed
	RF <- kT(data$temp(t), parms['k600']) * (osat(data$temp(t), data$P) - y)

	# ecosystem respiration; Fuß et al eq 4
	ER <- (parms['ER24_20'] / (60*24)) * (1.045^(data$temp(t) - 20))

	val <- (GPP + ER + RF) / data$z
	if(return_list) {
		return(list(val))
	} else {
		return(val)
	}
}

#' Predict a time series of dissolved oxygen concentration
#' @param initial initial dissolved oxygen concentration
#' @param times The times at which to solve the system
#' @param params a named vector of parameters; see details
#' @param data a list of data for the ode; see details
#' @param dt Time step for integration; only used if `method == 'euler'`
#' @param method The integration method to use; default is euler
#' @param gpp Logical, should GPP at each time step be returned as well?
#' @details Light and temperature time series will be approximated using linear interpolation
#' 		at the desired time steps
#'
#' `params` must be a vector including the following named items:
#' * `P1` $(W min g^{-1} O_2)$ inverse of the slope of a photosynthesis–irradiance curve at low
#' 		 light intensity
#' * `P2` $(m^2 min g^{-1} O_2)$ inverse maximum photosynthesis rate; can be zero to assume
#' 		  GPP is linear with light intensity instead of saturating
#' * `k600` coefficient of gas exchange for a gas with a Schmidt number of 600; (see [kT()])
#' * `ER24_20` daily ecosystem respiration rate, standardized at 20 degrees C
#'
#' `data` is a named list of (constant) data items, including the following:
#' * `PAR`  2-column data frame; first column is light, second is time of observation
#' * `temp` 2-column data frame; first column is temperature, second is time of observation
#' * `P`    pressure, in atmospheres
#' * `z`    Depth, in meters
#' @return Time series of dissolved oxygen concentrations
#' @export
oneStation_DOPredict <- function(initial, times, params, data, dt = 1,
			method=c('euler', 'lsoda'), gpp = FALSE) {
	warning("oneStation_DOPredict is deprecated; use DOCalibration(..., method = 'stan' instead)")
	method <- match.arg(method)
	if(method == "lsoda" && !(requireNamespace("deSolve")))
		stop("Package deSolve is required for method lsoda; please install it and try again")
	if(times[1] != 0)
		stop('first time step must be 0')
	if(method == 'euler' && sum(times %% dt) != 0)
		stop('dt must divide evenly into all values in times')

	dDOData <- list(
		PAR = approxfun(x = data$PAR[,2], y = data$PAR[,1], rule = 2),
		temp = approxfun(x = data$temp[,2], y = data$temp[,1], rule = 2),
		P = data$P,
		z = data$z)

	if(method == 'euler') {
		state <- numeric(length(times))
		state[1] <- statet <- initial
		t <- times[1]
		for(i in 2:length(state)) {
			while(t < times[i]) {
				statet <- statet + dt * oneStation_dDOdt(t, statet, params, dDOData)
				t <- t + dt
			}
			state[i] <- statet
		}
	} else {
		state <- deSolve::ode(initial, times = times, func = oneStation_dDOdt, parms = params,
			data = dDOData, return_list=TRUE)
		state <- state[,2]
	}
	if(gpp) {
		state <- cbind(DO = state, GPP = gpp(dDOData$PAR(times), params['P1'], params['P2']))
	}
	return(state)
}

#' Log posterior probability for the DO curve given empirical data
#' @param params A named vector of model parameters; see details
#' @param data A list of model data, see details
#' @param prior A list of prior hyperparameters; see details
#' @param ... Additional parameters to be passed to [DO_predict()]
#' @details #' `params` must be a vector including the following named items:
#' * `logP1` $(W min g^{-1} O_2)$ log(inverse of the slope of a photosynthesis–irradiance curve)
#' 		 at low light intensity
#' * `logP2` $(m^2 min g^{-1} O_2)$ log(inverse maximum photosynthesis rate)
#' * `logk600` coefficient of gas exchange for a gas with a Schmidt number of 600; (see [kT()])
#' * `logMinusER24_20` log(-1 * daily ecosystem respiration rate), standardized at 20 degrees C
#' * `logsd` log(Error standard deviation)
#'
#' `data` is a named list of (constant) data items, including the following:
#' * `DO` 2-column data frame; first column is dissolved oxygen, second is time of observation
#' * `PAR`  2-column data frame; first column is light, second is time of observation
#' * `temp` 2-column data frame; first column is temperature, second is time of observation
#' * `P`    pressure, in atmospheres
#' * `z`    Depth, in meters
#'
#' `prior` is a list of hyperparameters for the priors on parameters listed under `params`.
#' All priors are normal. Thus, each list item should be named (following the names from
#' `params`), and each item should be a vector of length two giving the mean and standard
#' deviation of the prior for that parameter. Note that each parameter is optional
#' (as is the entire list); missing items will get a minimally informative normal(0,10)
#'  distribution.
#' @return The unnormalized log probability of the model given the data
#' @export
oneStation_DOlogprob <- function(params, data, prior = list(), ...) {
	warning("oneStation_DOlogprob is deprecated; use DOCalibration(..., method = 'stan' instead)")
	# check for each parameter, assign default priors if not specified
	trParNames <- c('logP1', 'logP2', 'logk600', 'logMinusER24_20', 'logsd')
	parNames <- c('P1', 'P2', 'k600', 'ER24_20', 'sd')
	for(nm in trParNames) {
		if(! nm %in% names(prior))
			prior[[nm]] <- c(0,10)
	}

	initial <- data$DO[1,1]
	times <- data$DO[,2]

	# parameters are log transformed to make sampling nicer
	doParams <- oneStation_parTransform(params)
	predicted <- oneStation_DOPredict(initial, times, doParams, data)

	logprob <- sum(dnorm(data$DO[,1], predicted, doParams['sd'], log = TRUE)) +
		dnorm(params['logP1'], prior[['logP1']][1], prior[['logP1']][2], log = TRUE) +
		dnorm(params['logP2'], prior[['logP2']][1], prior[['logP2']][2], log = TRUE) +
		dnorm(params['logk600'], prior[['logk600']][1], prior[['logk600']][2], log = TRUE) +
		dnorm(params['logMinusER24_20'], prior[['logMinusER24_20']][1], prior[['logMinusER24_20']][2], log = TRUE) +
		dnorm(params['logsd'], prior[['logsd']][1], prior[['logsd']][2], log = TRUE)

	return(logprob)
}

#' Transfornm parameters for the one station model
#' @param params The vector of parameters, see [oneStation_DOlogprob()], or a matrix of
#' 		samples (in which case the transformation is applied to app samples)
#' @param reverse Should the reverse transformation be applied?
#' @details This function transforms the parameters of the one station model from the
#' 	scale at which the parameters are calibrated to the scale of modelling. If
#'  `reverse == TRUE` the opposite transformation is performed.
#' @return The transformed parameters
#' @keywords internal
oneStation_parTransform <- function(params, reverse = FALSE) {
	if(reverse) {
		if(is.matrix(params)) {
			logP1 <- log(params[,'P1'])
			logP2 <- log(params[,'P2'])
			logk600 <- log(params[,'k600'])
			logMinusER24_20 <- log(-1 * params[,'ER24_20'])
			logsd <- log(params[,'sd'])
			result <- cbind(logP1, logP2, logk600, logMinusER24_20, logsd)
		} else {
			logP1 <- log(params['P1'])
			logP2 <- log(params['P2'])
			logk600 <- log(params['k600'])
			logMinusER24_20 <- log(-1 * params['ER24_20'])
			logsd <- log(params['sd'])
			result <- c(logP1 = logP1, logP2 = logP2, logk600 = logk600,
				logMinusER24_20 = logMinusER24_20, logsd = logsd)
		}
	} else {
		if(is.matrix(params)) {
			P1 <- exp(params[,'logP1'])
			P2 <- exp(params[,'logP2'])
			k600 <- exp(params[,'logk600'])
			ER24_20 <- -1 * exp(params[,'logMinusER24_20'])
			sd <- exp(params[,'logsd'])
			result <- cbind(P1, P2, k600, ER24_20, sd)
		} else {
			P1 <- exp(params['logP1'])
			P2 <- exp(params['logP2'])
			k600 <- exp(params['logk600'])
			ER24_20 <- -1 * exp(params['logMinusER24_20'])
			sd <- exp(params['logsd'])
			result <- c(P1, P2, k600, ER24_20, sd)
			names(result) <- c("P1", "P2", "k600", "ER24_20", "sd")
		}
	}
	return(result)
}

#' Produce posterior simulations for DO curve and GPP
#' @param params Parameter posterior matrix returned (e.g.) from [DOCalibration()].
#' @param data Data list as required by [oneStation_DOPredict()].
#' @return list including predicted DO and GPP at each time step as well as the prediction times.
oneStation_DOSim <- function(params, data) {
	initial <- data$DO[1,1]
	times <- data$DO[,2]

	sims <- apply(params, 1, function(par) {
		c(oneStation_DOPredict(initial, times, par, data, dt = 1, gpp=TRUE))
	})
	list(DO = sims[1:length(times),], GPP = sims[(length(times)+1):nrow(sims),], times = times)
}


#' @export

d_rho <- function(rho, parms, z = 4) {
  with(parms, r* b* rho * (1 - rho/K) - m * rho - (a * rho * L)/(1 + a * h * rho) )
}

#' @export
noymeir <- list(
 defparms = list(
   m = 0.05,
   r = 1,
   b = 0.9,
   K = 0.9,
   a = 0.6,
   h = 100,
   L = 5
 ),
 odesys <- function (t, rho, parms = parms) {
   list( d_rho(rho, parms)  )
 }
)


#!/usr/bin/Rscript

cdfFromLogScaledCbf <- function()
{
    # This function fabricates a cdf from a
    # "Piecewise LogLinear Cumulative Byte Function" of a workload.
    # all you need to do is to specify sizeIntervals and cbf (cumulative byte
    # function) values in below. The assumption is that cbf is piecewise linear
    # function of log10(sizeIntervals) with breakpoints defined in sizeIntervals
    # and cbf vectors.

    # monotonically increasing vector of msg sizes
    #sizeIntervals <- c(1, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288, 3529904)
    sizeIntervals <- c(1, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288, 15158197)

    # monotonically increaing cbf (cumulative byte function) vector. First element
    # is always 0 and last element is always one. Each cbf value in this vector
    # corresponds to a size in the sizeIntervals vector.
    #cbf <- c(0, .0016, .0160256, .01923077, .07051282, .4455128, .525641, .6025641, .6987179, .7403846, .7532051, .7660256, .7884615, .900641, .9455128, .9519231, 1)
    cbf <- c(0, .0001205128, .001010256, .0070385, .01602564, .05128205, .07051282, .09294872, .1185897, .1762821, .2564103, .3012821, .3461538, .4679487, .5512321, .6826923, 1)

    multipLiers <- (cbf[2:length(cbf)] - cbf[1:length(cbf)-1]) *
        (1/sizeIntervals[1:length(cbf)-1] - 1/sizeIntervals[2:length(cbf)]) /
        (log10(sizeIntervals[2:length(cbf)]) - log10(sizeIntervals[1:length(cbf)-1]))
    avgSize <- log(10)/sum(multipLiers)
    cdfRanges <- cumsum(multipLiers)/sum(multipLiers)
    cdfConsts <- c(0, cdfRanges[1:length(cdfRanges)-1])

    #create cdf sample vector and find the corresponding sizes.
    #cdf sample vector must be sorted: cdf <- seq(0.001, 1, 0.001)
    cdf <- c()
    cdfBounds <- c(0, cdfRanges)
    numElemDouble = 3
    for (ind in seq(1, length(cdfRanges)))
    {
        numElemDouble <- numElemDouble*1.3
        numElem <- as.integer(numElemDouble)
        cdf <- c(cdf, seq(cdfBounds[ind], cdfBounds[ind+1], (cdfBounds[ind+1] - cdfBounds[ind])/numElem))
    }
    cdf <- sort(unique(cdf))

    #Corresponding sizes for cdf values
    sizes <- c()
    for (cdfVal in cdf)
    {
        ind <- match(1, findInterval(cdfRanges, cdfVal))
        invMsgSize <- (1 / sizeIntervals[ind]) -
            ((cdfVal - cdfConsts[ind]) * log(10) *
            (log10(sizeIntervals[ind+1]) - log10(sizeIntervals[ind])) /
            (cbf[ind+1]-cbf[ind]) / avgSize)
        sizes <- c(sizes, 1/invMsgSize)
    }

    # round the sizes to integers and remove possible duplicates that rounding might
    # have caused.
    sizes <- round(sizes)
    unik <- !duplicated(sizes) #logical vector of unique values
    ind <- seq_along(sizes)[unik] #indices of unique values
    cdf <- cdf[ind]
    sizes <- sizes[ind]
    pdf <- cdf - c(0, cdf[1:length(cdf)-1])
    avgSize <- sum(pdf*sizes)

    filename = "Google_AllRPC.txt"
    write(avgSize, file=filename)
    df <- data.frame(sizes, cdf)
    write.table(df, file=filename, row.names=FALSE, col.names=FALSE, append=TRUE)
}

cdfFromCdfSamples <- function()
{
    # This function fabricates a cdf from a
    # "Piecewise LogLinear CDF Function" of a workload.
    # all you need to do is to specify sizeIntervals and cdf
    # values in below. The assumption is that cdf is piecewise linear
    # function of log10(sizeIntervals) with breakpoints defined in sizeIntervals
    # and cdf vectors.

    # monotonically increasing vector of msg sizes
    sizeIntervals <- c()

    # monotonically increaing cdf vector. First element
    # is always 0 and last element is always one. Each cdf value in this vector
    # corresponds to a size in the sizeIntervals vector.
    cdfIntervals <- c()

    # For Facebook Hadoop All
    sizeIntervals <- c(50, 92 ,217 ,271 ,300 ,326 ,376 ,425 ,480 ,600 ,679 ,737 ,800 ,885 ,1042 ,1277 ,
        1413 ,1534 ,1664 ,2214 ,3396 ,5210 ,7830 ,31946 ,36844 ,39973 ,44260 ,47050 ,51046 ,62583 ,
        70722 ,75180 ,81565 ,102058 ,115331 ,133013 ,156563 ,184284 ,212546 ,240177 ,288520 ,470507 ,
        736642 ,940700 ,1769244 ,2169118 ,10000000)
    cdfIntervals <- c(0, 0.0074 ,0.0149 ,0.0409 ,0.0520 ,0.1375 ,0.2026 ,0.2230 ,0.2639 ,0.4591 ,
        0.5167 ,0.5520 ,0.5762 ,0.5912 ,0.6097 ,0.6282 ,0.6394 ,0.6468 ,0.6673 ,0.6747 ,0.6840 ,
        0.6933 ,0.7082 ,0.7212 ,0.7305 ,0.7398 ,0.7584 ,0.7788 ,0.8160 ,0.8234 ,0.8327 ,0.8420 ,
        0.8717 ,0.8885 ,0.8978 ,0.9070 ,0.9164 ,0.9257 ,0.9349 ,0.9442 ,0.9535 ,0.9628 ,0.9684 ,
        0.9721 ,0.9814 ,0.9870 ,1.0000)

    # For FacebookWebServer
    #sizeIntervals <- c(50, 84 ,153 ,301 ,500 ,639 ,800 ,900 ,1000 ,1085 ,1357 ,1534 ,
    #    1664 ,1843 ,2000 ,3000 ,4000 ,5000 ,6000 ,7000 ,8000 ,9000 ,10000 ,14140 ,20000 ,
    #    24512 ,30000 ,34660 ,40000 ,45171 ,50000 ,60000 ,70000 ,80000 ,100000 ,144310 ,
    #    200000 ,260573 ,300000 ,353730 ,400000 ,442606 ,565213 ,778003 ,1000000)
    #
    #cdfIntervals <- c(0.0 ,0.061 ,0.126 ,0.215 ,0.226 ,0.2407 ,0.2537 ,0.2667 ,0.2926 ,
    #    0.3130 ,0.3352 ,0.3500 ,0.3722 ,0.3963 ,0.4093 ,0.4796 ,0.5259 ,0.5630 ,0.5889 ,
    #    0.6111 ,0.6260 ,0.6407 ,0.6519 ,0.6852 ,0.7148 ,0.7315 ,0.7463 ,0.7574 ,0.7667 ,
    #    0.7759 ,0.7815 ,0.7926 ,0.8 ,0.8074 ,0.8148 ,0.8241 ,0.8574 ,0.8944 ,0.9222 ,
    #    0.9556 ,0.9741 ,0.9833 ,0.9907 ,0.9963 ,1.0)

    # For Facebook CacheFollower Intracluster
    #sizeIntervals <- c(50 ,68 ,144 ,266 ,340 ,490 ,577 ,639 ,707 ,900 ,1042 ,1110 ,1303 ,1414 ,1440 ,2000 ,
    #    2450 ,3000 ,3466 ,4000 ,5000 ,6000 ,8000 ,10000 ,13855 ,16643 ,18057 ,25531 ,28270 ,36101 ,44260 ,
    #    62583 , 72178 ,81565 ,92173 ,102060 ,110726 ,127700 ,141398 ,156563 ,169860 ,184283 ,199933 ,212537 ,
    #    225934 ,245121 ,260573 ,276998 ,326042 ,391670 ,531696 ,588724 ,1385457 ,1503111 ,1597864 ,1681372 ,
    #    1787361 ,1842836 ,1959004 ,2125366 ,2213771 ,2329467 ,2553169 ,2827011 ,3006219 ,3260426 ,3837698)
    #cdfIntervals <- c(0.0 ,0.0204 ,0.0409 ,0.0539 ,0.1803 ,0.2770 ,0.3048 ,0.3178 ,0.3532 ,0.3643 ,0.3773 ,
    #    0.3996 ,0.4052 ,0.4126 ,0.4238 ,0.4368 ,0.4461 ,0.4535 ,0.4572 ,0.4610 ,0.4665 ,0.4703 ,0.4758 ,0.4777 ,
    #    0.4833 ,0.4870 ,0.4888 ,0.4944 ,0.5000 ,0.5037 ,0.5091 ,0.5186 ,0.5223 ,0.5260 ,0.5297 ,0.5335 ,0.5390 ,
    #    0.5465 ,0.5558 ,0.5651 ,0.5743 ,0.5836 ,0.5923 ,0.6022 ,0.6115 ,0.6208 ,0.6301 ,0.6394 ,0.6580 ,0.6766 ,
    #    0.6952 ,0.7138 ,0.7200 ,0.7230 ,0.7416 ,0.7602 ,0.7787 ,0.7974 ,0.8160 ,0.8346 ,0.8532 ,0.8717 ,0.9276 ,
    #    0.9740 ,0.9888 ,0.9944 ,1.0)

    cdfRanges <- cdfIntervals[2:length(cdfIntervals)]
    cdfConsts <- cdfIntervals[1:length(cdfIntervals)-1]

    #create cdf sample vector and find the corresponding sizes.
    #cdf sample vector must be sorted: cdf <- seq(0.001, 1, 0.001)
    cdf <- c()
    for (ind in seq(1, length(cdfRanges)))
    {
        cdf <- c(cdf, seq(cdfIntervals[ind], cdfIntervals[ind+1], (cdfIntervals[ind+1] - cdfIntervals[ind])/5))
    }
    cdf <- sort(unique(cdf))

    #Corresponding sizes for cdf values
    sizes <- c()
    for (cdfVal in cdf)
    {
        ind <- match(1, findInterval(cdfRanges, cdfVal))
        logMsgSize <- (cdfVal - cdfConsts[ind]) *
            (log10(sizeIntervals[ind+1])-log10(sizeIntervals[ind])) /
            (cdfRanges[ind]-cdfConsts[ind]) + log10(sizeIntervals[ind])
        sizes <- c(sizes, 10**(logMsgSize))
    }

    # round the sizes to integers and remove possible duplicates that rounding might
    # have caused.
    sizes <- round(sizes)
    unik <- !duplicated(sizes) #logical vector of unique values
    ind <- seq_along(sizes)[unik] #indices of unique values
    cdf <- cdf[ind]
    sizes <- sizes[ind]
    pdf <- cdf - c(0, cdf[1:length(cdf)-1])
    avgSize <- sum(pdf*sizes)

    filename = "Facebook_HadoopDist_All.txt"
    write(avgSize, file=filename)
    df <- data.frame(sizes, cdf)
    write.table(df, file=filename, row.names=FALSE, col.names=FALSE, append=TRUE)
}

cdfFromLogScaledCbf()