##################################################################
#
#
#                              >>> Geodesy In R <<<
#
#    This file contains a series of geodetic functions to aid in performing
#    spatial analysis across large distances (100s of km). Note: R Trig.
#    functions take arguments in radians. 1 degree = pi/180 radians
#
# original source by Darren Gillis and others
#
#	Various Chart Datum Parameters:
#
#	Name           Major Axis, a (km)   Flattening (f)
#	WGS84          6378.13700	          1/298.257223563
#	GRS80/NAD83    6378.13700	          1/298.257222101
#	WGS66          6378.145             1/298.25
#	GRS67/IAU68    6378.16000           1/298.2472
#	WGS72          6378.135             1/298.26
#	Krasovsky      6378.245             1/298.3
#	Clarke66/NAD27 6378.2064            1/294.9786982138
#
#	Coordinate Systems and Map Projections, D. H. Maling (Pergamon 1992)
#	Summarized by Ed Williams (http://williams.best.vwh.net/index.html)
#
#     Earth Radius as geometric mean of major & minor radia (WGS84)
#     Nautical Miles per degree as 57.29
#
#
#	Note: Radius of minor axis = a - a*f
#
# test data
#      point     = data.frame(lat=-30, lon=150)
#      locations = data.frame(lat=c(-31, -35, -40), lon=c(150, 151, 151))
#      vincenty distances = 110.8609, 562.3759, 1113.1415
#      great circle =       111.1327  563.5021  1115.0260
##################################################################

##################################################################
# vincenty method .. more accurate
# the original fortran code and documentation is copied at the bottom
##################################################################

vincenty = function (loc1, loc2, a, f) {

  ## for history and equations:: see https://github.com/OSGeo/proj.4/wiki/GeodesicCalculations
  warning( "Deprecated. Consider using geosphere::distVincetyEllipsoid " )

  names(loc1) = c("lon", "lat")
  names(loc2) = c("lon", "lat")

      eps = 0.5e-13
      pi = 3.1415926535897932384626433832795
      rad = 0.0174532925199432957692369076848861
      r = 1.0-f

#     convert decimal degrees to radians

      rloc1 = rad*loc1
      rloc2 = rad*loc2

      tu1 = r * sin(rloc1$lat) / cos(rloc1$lat)
      tu2 = r * sin(rloc2$lat) / cos(rloc2$lat)

      cu1 = 1. / sqrt(tu1 * tu1 + 1. )
      su1 = cu1 * tu1
      cu2 = 1. / sqrt(tu2 * tu2 + 1. )

      s =   cu1 * cu2
      baz = s * tu2
      faz = baz * tu1
      dlon = (rloc2$lon - rloc1$lon)
      x=dlon

      q = c(1:dim(loc2)[1]) # get it started into the loop

      sx = NULL
      cx = NULL
      tu1 = NULL
      tu2 = NULL
      sy = NULL
      cy = NULL
      y = NULL
      sa = NULL
      c2a = NULL
      cz = NULL
      e = NULL
      c = NULL

  		repeat {
  			sx[q] = sin(x[q])
        cx[q] = cos(x[q])
        tu1[q] = cu2[q] * sx[q]
        tu2[q] = baz[q] - su1 * cu2[q] * cx[q]
        sy[q] = sqrt(tu1[q] * tu1[q] + tu2[q] * tu2[q])
        cy[q] = s[q] * cx[q] + faz[q]
        y[q] = atan2(sy[q], cy[q])
        sa[q] = s[q] * sx[q] / sy[q]
        c2a[q] = - sa[q] * sa[q] + 1.
        nz = intersect(which(c2a != 0), q)
        cz[q] = faz[q] + faz[q]
        cz[nz] = -cz[nz] / c2a[nz] + cy[nz]
        e[q] = cz[q] * cz[q] * 2 - 1
        c[q] = ((-3. * c2a[q] + 4.) * f + 4.) * c2a[q] * f / 16.
        d = x
        x[q] = ((e[q] * cy[q] * c[q] + cz[q]) * sy[q] * c[q] + y[q]) * sa[q]
        x[q] = (1. - c[q]) * x[q] * f + dlon[q]
        q = which(abs(d - x) > eps)
        if (length(q) < 1) break
      }

      #faz = atan2(tu1, tu2)
      #baz = atan2(cu1 * sx, baz * cx - su1 * cu2) + pi
      x = sqrt( (1. / r / r - 1.) * c2a + 1. ) + 1.
      x = (x - 2.) / x
      c = 1. - x
      c = (x * x / 4. + 1.) / c
      d = (0.375 * x * x - 1.) * x
      x = e * cy
      s = 1. - e - e
      s = ((((sy * sy * 4. - 3.) * s * cz * d / 6. - x ) * d / 4. + cz) * sy * d + y) * c * a * r

      s [!is.finite(s)] = 0

      #faz = faz / rad # these are azimuths .. not needed
      #baz = baz / rad

      return(s)

      }

##################################################################
#~ C *** SOLUTION OF THE GEODETIC INVERSE PROBLEM AFTER T.VINCENTY
#~ C *** MODIFIED RAINSFORD'S METHOD WITH HELMERT'S ELLIPTICAL TERMS
#~ C *** EFFECTIVE IN ANY AZIMUTH AND AT ANY DISTANCE SHORT OF ANTIPODAL
#~ C *** STANDPOINT/FOREPOINT MUST NOT BE THE GEOGRAPHIC POLE
#~ C
#~ C *** A IS THE SEMI-MAJOR AXIS OF THE REFERENCE ELLIPSOID
#~ C *** F IS THE FLATTENING (NOT RECIPROCAL) OF THE REFERNECE ELLIPSOID
#~ C *** LATITUDES AND LONGITUDES IN RADIANS POSITIVE NORTH AND EAST
#~ C *** FORWARD AZIMUTHS AT BOTH POINTS RETURNED IN RADIANS FROM NORTH
#~ C
#~ C *** PROGRAMMED FOR CDC-6600 BY LCDR L.PFEIFER NGS ROCKVILLE MD 18FEB75
#~ C *** MODIFIED FOR IBM SYSTEM 360 BY JOHN G GERGEN NGS ROCKVILLE MD 7507
#~ C
#~ C *** Modified for R by D.Gillis Zoology University of Manitoba 16JUN03
#~ C     Replaced common blocks for constants with DATA statements.  Datum
#~ C     parameters moved from common block to subroutine arguements.
#~ C
#~ C *** Input Variables: DLAT1,DLON1 - initial fix (P1) in degrees
#~ C                          (latitude, north +) (longitude east +)
#~ C                      DLAT2,DLON2 - destination fix (P2) in degrees
#~ C
#~ C                      Ellipsoid (spheroid model, eg. WGS84)
#~ C                         A   - radius of major axis in distance units
#~ C                         F   - flattening factor
#~ C
#~ C *** Output Variables: FAZ - azimuth of the geodesic (P1 to P2)
#~ C                       BAZ - azimuth of the geodesic (P2 to P1)
#~ C                       S   - spheroidal distance = length of the geodesic
#~ C                             in distance units
#~ C
#~ C *** After Vincenty,T. 1975. Direct and inverse solutions of geodesics
#~ C           on the ellipsoid with application of nested equations. Survey
#~ C           Review 23(176):88-94.
#~ C
#~ C
#~ C     These routines were compiled in Windows 98 (DOS Window) using the
#~ C     Gnu FORTRAN complier: g77 --share -o geodesy.dll marspat.for
#~ C     creating:             geodesy.dll
#~ C
#~ C
#~ C
#
#  original code ("geodesy.for") is as follows:
#     note the return of azimuths has been turned off
#
      #~ SUBROUTINE INVER1(DLAT1,DLON1,DLAT2,DLON2,A,F,FAZ,BAZ,S)
      #~ IMPLICIT DOUBLE PRECISION (A-H,O-Z)
#~ C     COMMON/CONST/PI,RAD - original code
#~ C     COMMON/ELIPSOID/A,F - original code
      #~ DATA EPS/0.5D-13/
      #~ DATA PI/3.1415926535897932384626433832795D0/
      #~ DATA RAD/0.0174532925199432957692369076848861D0/
      #~ R=1.D0-F
#~ C
#~ C     Convert decimal degrees to radians
#~ C
      #~ GLAT1=DLAT1*RAD
      #~ GLON1=DLON1*RAD
      #~ GLAT2=DLAT2*RAD
      #~ GLON2=DLON2*RAD
#~ C
      #~ TU1=R*DSIN(GLAT1)/DCOS(GLAT1)
      #~ TU2=R*DSIN(GLAT2)/DCOS(GLAT2)
      #~ CU1=1./DSQRT(TU1*TU1+1.)
      #~ SU1=CU1*TU1
      #~ CU2=1./DSQRT(TU2*TU2+1.)
      #~ S=CU1*CU2
      #~ BAZ=S*TU2
      #~ FAZ=BAZ*TU1
      #~ X=GLON2-GLON1
  #~ 100 SX=DSIN(X)
      #~ CX=DCOS(X)
      #~ TU1=CU2*SX
      #~ TU2=BAZ-SU1*CU2*CX
      #~ SY=DSQRT(TU1*TU1+TU2*TU2)
      #~ CY=S*CX+FAZ
      #~ Y=DATAN2(SY,CY)
      #~ SA=S*SX/SY
      #~ C2A=-SA*SA+1.
      #~ CZ=FAZ+FAZ
      #~ IF(C2A.GT.0.)CZ=-CZ/C2A+CY
      #~ E=CZ*CZ*2.-1.
      #~ C=((-3.*C2A+4.)*F+4.)*C2A*F/16.
      #~ D=X
      #~ X=((E*CY*C+CZ)*SY*C+Y)*SA
      #~ X=(1.-C)*X*F+GLON2-GLON1
      #~ IF(DABS(D-X).GT.EPS) GOTO 100
      #~ FAZ=DATAN2(TU1,TU2)
      #~ BAZ=DATAN2(CU1*SX,BAZ*CX-SU1*CU2)+PI
      #~ X=DSQRT((1./R/R-1.)*C2A+1.)+1.
      #~ X=(X-2.)/X
      #~ C=1.-X
      #~ C=(X*X/4.+1.)/C
      #~ D=(0.375*X*X-1.)*X
      #~ X=E*CY
      #~ S=1.-E-E
      #~ S=((((SY*SY*4.-3.)*S*CZ*D/6.-X)*D/4.+CZ)*SY*D+Y)*C*A*R
      #~ FAZ=FAZ/RAD
      #~ BAZ=BAZ/RAD
      #~ END
#
# example method of access to shared library from R
#
#~ dyn.load("~/src/grids/geodesy.lib")
	#~ inver1<- function (Lat1, Lon1, Lat2, Lon2) {
	              #~ a<-6378137.00          # WGS84 major axis
	              #~ f<-1/298.257223563     # WGS84 flattening parameter
                    #~ .Fortran("inver1",as.double(Lat1),as.double(Lon1),
                                      #~ as.double(Lat2),as.double(Lon2),
                                      #~ as.double(a),as.double(f),
                                      #~ as.double(0),as.double(0),
                                      #~ as.double(0)) [[9]]
      #~ }

  #~ GDvincenty <- function (Lat1, Lon1, Lat2, Lon2) {
 		#~ n = length(Lat1)
	  #~ ans = inver1(Lat1,Lon1,Lat2,Lon2)

    #~ if (n>1) {
      #~ for (i in 2:n) {
      	#~ ans<-c(ans, inver1(Lat1[i],Lon1[i],Lat2[i],Lon2[i]))
      #~ }
    #~ }
    #~ ans
	#~ }

#
##################################################################


ecent_coords <- function(pts){ # pts je vektor sa 4 elementa x, y, e_cent, seed.
  e_cent <- pts[3]/1000
  sigma <- matrix(c(e_cent^2, 0, 0, e_cent^2),2)
  set.seed(pts[4])
  pt_coords <- mvrnorm(1, pts[1:2], sigma)
  return(pt_coords)
}

ni3 <- function(x1, y1, x2, y2){
  ## body
  dEasting <- as.numeric(x2 - x1)
  dNorthing <- as.numeric(y2 - y1)
  atg <- ifelse(dNorthing < 0, atan(dEasting/dNorthing)*180/pi + 180, atan(dEasting/dNorthing)*180/pi)
  ang <- ifelse(atg < 0, atg + 360, atg)
  return(ang)
}

sim_st <- function(survey.net.st, red){
  prj <- st_crs(survey.net.st)
  ro <- 180/pi*3600
  st_data <- survey.net.st %>% st_drop_geometry() %>% .[1, c("x_from", "y_from", "e_cent_from", "seed_from")] %>% as.numeric()
  st_coords_e <- ecent_coords(pts = st_data)
  e_cent_from <- dist(pt1_coords = st_data[1:2], pt2_coords = st_coords_e)*1000

  target_coords_e <- survey.net.st %>% st_drop_geometry() %>% .[, c("x_to", "y_to", "e_cent_to", "seed_to")] %>%  # TODO: seed ne radi.
    apply(., 1, function(x) ecent_coords(pts = as.numeric(x))) %>% t()

  e_cent_to <- data.frame((survey.net.st %>% st_drop_geometry() %>% .[, c("x_to", "y_to")] - target_coords_e)*1000) %>%
    dplyr::rename(e_x = x_to, e_y = y_to) %>%
    dplyr::mutate(e_cent_to = sqrt(e_x^2+e_y^2)) %>%
    .$e_cent_to

  survey.net.st$e_cent_from <- e_cent_from
  survey.net.st$e_cent_to <- e_cent_to

  survey.net.st <- survey.net.st %>%
    dplyr::mutate(x_from = st_coords_e[1],
                  y_from = st_coords_e[2],
                  x_to = target_coords_e[, 1],
                  y_to = target_coords_e[, 2],
                  d = sqrt((x_from-x_to)^2+(y_from-y_to)^2),
                  sd_e_cent_from = sqrt((e_cent_from^2*ro^2)/(2*(d*1000)^2)),
                  sd_e_cent_to = sqrt((e_cent_to^2*ro^2)/(2*(d*1000)^2)),
                  sd_e_cent_dist_from = sqrt(e_cent_from^2/2),
                  sd_e_cent_dist_to = sqrt(e_cent_to^2/2),
                  ni1 = ni3(x_from, y_from, x_to, y_to),
                  z = 360-Hz0,
                  Hz_sim = ifelse(ni1 >= Hz0, z+ni1-360, z+ni1),
                  sd_hz_sq = (e_air^2 + e_focus^2 + sd_Hz^2) + sd_e_cent_from^2 + sd_e_cent_to^2,
                  sd_dist_sq = sd_e_cent_dist_from^2 + sd_e_cent_dist_to^2 + sd_dist^2)

  if(red == TRUE){
    survey.net.st <- survey.net.st %>%
      # dplyr::mutate_at(., .vars = "Hz_sim", ~Hz_sim - Hz_sim[1]) %>%
      dplyr::mutate_at(., .vars = c("sd_hz_sq", "sd_dist_sq"), ~replace(., is.na(.), 0)) %>%
      dplyr::mutate(Hz_sim = ifelse(Hz_sim < 0, Hz_sim + 360, Hz_sim),
                    Hz_sim = Hz_sim - Hz_sim[1],
                    Hz_sim = ifelse(Hz_sim < 0, Hz_sim + 360, Hz_sim), # dplyr::case_when(Hz_sim < 0 ~ Hz_sim + 360, Hz_sim >= 0 ~ Hz_sim),
                    sd_hz_sq = ifelse(is.na(sd_hz_sq), NA, sd_hz_sq + sd_hz_sq[1]),
                    sd_hz_sample = rnorm(dim(survey.net.st)[1], 0, sqrt(sd_hz_sq)),
                    Hz_sim = Hz_sim + sd_hz_sample/3600,
                    Hz = ifelse(round(Hz_sim, 0) == 0, round(Hz_sim, 0), Hz_sim),
                    Hz = pmax(Hz, 0),
                    Hz = ifelse(is.na(sd_Hz), NA, Hz),  #sa ifelse se moze i ostalo resiti da ne stavlja 0 tamo za sd_hz_sq,
                    HzD = floor(Hz), HzM = floor((Hz-HzD)*60), HzS = round(((Hz-HzD)*60-HzM)*60, 1),
                    sd_dist_sample = rnorm(dim(survey.net.st)[1], 0, sqrt(sd_dist_sq)),
                    dist_sim = d + sd_dist_sample/1000,
                    HD = round(dist_sim, 4),
                    HD = ifelse(is.na(sd_dist), NA, HD)) %>%
      as.data.frame() %>%
      dplyr::arrange(ID, Hz) %>%
      st_sf(., crs = prj)
  }else{
    survey.net.st <- survey.net.st %>%
      dplyr::mutate(Hz_sim = ifelse(Hz_sim < 0, Hz_sim + 360, Hz_sim),
                    sd_hz_sample = rnorm(dim(survey.net.st)[1], 0, sqrt(sd_hz_sq)),
                    Hz_sim = Hz_sim + sd_hz_sample/3600,
                    Hz = pmax(Hz, 0),
                    HzD = floor(Hz), HzM = floor((Hz-HzD)*60), HzS = round(((Hz-HzD)*60-HzM)*60, 1),
                    sd_dist_sample = rnorm(dim(survey.net.st)[1], 0, sqrt(sd_dist_sq)),
                    dist_sim = d + sd_dist_sample/1000,
                    HD = round(dist_sim, 4),
                    HD = ifelse(is.na(sd_dist), NA, HD)) %>%
      as.data.frame() %>%
      dplyr::arrange(ID, Hz) %>%
      st_sf(., crs = prj)
  }
  return(survey.net.st)
}

#survey.net = mreza_s; red = TRUE; Hz0 = NA; seed = NULL; type = list("dms", "dec", "rad")

sim_snetobs <- function(survey.net, red = TRUE, Hz0 = NA, seed = NULL, type = list("dms", "dec", "rad")){

  survey.net[[2]] <- survey.net[[2]] %>% dplyr::mutate_at(., .vars = c("e_cent_from", "e_cent_to", "e_focus", "e_air"), ~replace(., is.na(.), 0))

  if(!is.na(Hz0)){
    if(length(Hz0) == 1) Hz0 <- rep(Hz0, length(unique(survey.net[[2]]$from)))
    if(length(Hz0) != length(unique(survey.net[[2]]$from))) stop("Hz0 must be of length either 1 or number of stations")
  }else{
    Hz0 <- sample(1:359, size = length(unique(survey.net[[2]]$from)))
  }

  if(!is.null(seed)){
    if(length(seed) == 1) seed <- rep(seed, length(unique(survey.net[[2]]$from)))
    if(length(seed) != length(unique(survey.net[[2]]$from))) stop("seed must be of length either 1 or the number of stations")
  }else{
    seed <- runif(length(unique(survey.net[[2]]$from)),0,100)#
  }

  nto <- survey.net[[2]] %>% st_drop_geometry() %>% dplyr::mutate(from = as_factor(from)) %>% group_by(from) %>% summarize(n = n())
  survey.net[[2]] <- survey.net[[2]] %>%
    group_by(from) %>%
    mutate(id = row_number()) %>% ungroup() %>%
    dplyr::mutate(Hz0 = rep(Hz0, nto$n),
                  Hz_sim = NA,
                  dist_sim = NA,
                  seed_from = rep(seed, nto$n),
                  seed_to = seed_from + id)

  survey.net[[2]] <- survey.net[[2]] %>% split(., f = factor(.$from)) %>% lapply(., function(x) sim_st(survey.net.st = x, red = red)) %>% do.call(rbind,.) %>% dplyr::arrange(ID)

  # fixed_points <- survey.net[[1]] %>% dplyr::filter(FIX_2D | FIX_1D) %>% .$Name
  #
  # if(length(fixed_points) > 1){
  #   survey.net[[2]][survey.net[[2]]$from %in% fixed_points & survey.net[[2]]$to %in% fixed_points, "HD"] <- NA
  # }

  return(survey.net)
}

#hz = mreza_sim$observations$Hz[2]; sd_Hz = mreza_sim$observations$sd_Hz[2]; e_colim = 15; sd_e_colim = 0.5; ng = 1; faces = "f1f2"

sim_face_hz <- function(hz, sd_Hz, e_colim, sd_e_colim, ng, faces = list("mean", "f1f2"), sd_sd_Hz, seed = NULL, hz_of = TRUE){
  faces = faces[1]
  if(faces == "f1f2"){
    set.seed(seed)
    hz_f1 <- hz - (rnorm(1, e_colim, sd_e_colim))/3600 + rnorm(1, 0, sd_Hz)*sqrt(2)/3600
    if(hz_f1 < 0){hz_f1 + 360}
    if(hz_f1 >= 180){
      set.seed(seed)
      hz_f2 <- (hz_f1 - 180) + (rnorm(1, e_colim, sd_e_colim))/3600 + rnorm(1, 0, sd_Hz)*sqrt(2)/3600
    }else{
      set.seed(seed)
      hz_f2 <- (hz_f1 + 180) + (rnorm(1, e_colim, sd_e_colim))/3600 + rnorm(1, 0, sd_Hz)*sqrt(2)/3600
    }
    hz <- data.frame(hz_f1 = ifelse(hz_f1 < 0, hz_f1 + 360, hz_f1) , hz_f2 = ifelse(hz_f2 < 0, hz_f2 + 360, hz_f2))
  }
  return(hz)
}

#a <- sim_face_hz(hz = mreza_sim$observations$Hz[1], sd_Hz = mreza_sim$observations$sd_Hz[1], e_colim = 15, sd_e_colim = 0.5, ng = 1, faces = "f1f2")
#a
#(a[2]+180-a[1])*3600


require(ggplot2)
require(scales)
SL = 32
p = 3

f = function(x,k,l,SL) { 1 - (1 - (1 - x/SL)^k)^l }

K = function(l,p,alpha,L) {round(log(1-(1-alpha)^(1/l))/log(1-p/L));}

K(4,2,0.95,10)

alpha=0.95
ggplot(data.frame(x = c(1, 3*p)), aes(x)) + theme_classic()+
  mapply(function( L) {
    stat_function(fun = f, args = list(k = K(l=L,p=p,alpha=alpha,L=SL), l=L, SL=SL), aes(color=paste(L,K(l=L,p=p,alpha=alpha,L=SL),sep="/")))
  },L=c(7:14))+
  geom_vline(xintercept = p,color="black",linetype=2)+
  geom_hline(yintercept = alpha,color="black",linetype=2)+
  scale_color_brewer(palette = "Spectral",name="l/h")+ylab("probability of a match")+xlab("Hamming distance")+scale_linetype(name="l")
ggsave("clus-95.pdf",width=5,height = 4)

alpha=0.1
ggplot(data.frame(x = c(1, 3*p)), aes(x)) + theme_classic()+
  mapply(function( L) {
    stat_function(fun = f, args = list(k = K(l=L,p=p,alpha=alpha,L=SL), l=L, SL=SL), aes(color=paste(L,K(l=L,p=p,alpha=alpha,L=SL),sep="/")))
  },L=c(2:8))+
  geom_vline(xintercept = p,color="black",linetype=2)+
  geom_hline(yintercept = alpha,color="black",linetype=2)+
  scale_color_brewer(palette = "Spectral",name="l/h")+ylab("probability of a match")+xlab("Hamming distance")+scale_linetype(name="l")
ggsave("clus-10.pdf",width=5,height = 4)

alpha=0.1
ggplot(data.frame(x = c(1, 3*p)), aes(x)) + theme_classic()+
  mapply(function( L) {
    stat_function(fun = function(x,k,l,SL) (150-SL+1)*f(x,k,l,SL), args = list(k = K(l=L,p=p,alpha=alpha,L=SL), l=L, SL=SL), aes(color=paste(L,K(l=L,p=p,alpha=alpha,L=SL),sep="/")))
  },L=c(2:8))+
  geom_vline(xintercept = 4,color="black",linetype=2)+
  geom_hline(yintercept = 3,color="black",linetype=2)+
  scale_color_brewer(palette = "Dark2",name="l/k")+ylab("probability of a match")+xlab("distance")+scale_linetype(name="l")

alpha=0.1
ggplot(data.frame(x = c(1, 16)), aes(x)) + theme_classic()+
  mapply(FUN=function( L, K) {
    stat_function(fun = function(x,k,l,SL) (1+150-SL)*f(x,k,l,SL),
                  args = list(k = K, l=L, SL=SL),
                  aes(color=as.factor(K),linetype=as.factor(L)))
  },L=rep(c(2),each=5), K=c(14))+
  #geom_vline(xintercept = 3,color="black",linetype=2)+
  #geom_vline(xintercept = SL*0.4,color="black",linetype=2)+
  #geom_hline(yintercept = 90,color="black",linetype=2)+
  scale_y_continuous(lim=c(0.1,100))+
  scale_linetype_manual(values=c(1,2,3),name=expression(h))+
  scale_x_continuous(labels=function(x) paste(x,percent(x/SL),sep="\n"))+
  scale_color_manual(name="l",values = c("black","gray80"))+
  ylab("Expected number of kmer matches per 150bp read")+
  xlab("Hamming distance")+
  #stat_function(fun = function(x) (1+150-SL)*f(x,35-7,1,35)/2, color="blue",linetype=1)+
  geom_vline(xintercept = 3,color="red",linetype=3)+
  geom_vline(xintercept = 10,color="red",linetype=3)+
  #theme(legend.position = "none")+
ggsave("exp-def.pdf",width=5,height = 4)


#' @title
#' Select ranks with cross validation
#'
#' @description
#' This function selects the optimal ranks \code{(r, rx, ry)} using a cross
#' validation procedure. The blockwise coordinate descent algorithm is used to fit
#' the model with any combinations of \code{(r, rx, ry)}.
#'
#' @usage
#' NRRR.cv(Y, X, nfold = 10, norder = NULL, Ag0 = NULL, Bg0 = NULL,
#'         jx, jy, p, d, n, maxiter = 300, conv = 1e-4,
#'         method = c('RRR','RRS')[1], lambda=0,
#'         dimred = c(TRUE,TRUE,TRUE), rankfix = NULL, xrankfix = NULL,
#'         yrankfix = NULL, lang=c('R','Rcpp')[1])
#'
#' @param Y response matrix of dimension n-by-jy*d.
#' @param X design matrix of dimension n-by-jx*p.
#' @param nfold the number of folds used in cross validation. Default is 10.
#' @param norder a vector of length n that assigns samples to multiple folds for cross validation.
#' @param Ag0 an initial estimator of matrix U. If \code{Ag0 = NULL} then generate it
#'            by \code{\link{NRRR.ini}}. Default is NULL. If \code{lang = 'Rcpp'},
#'            then \code{Ag0} is automatically generated by \code{\link{NRRR.ini}}.
#' @param Bg0 an initial estimator of matrix V, if \code{Bg0 = NULL} then generate it
#'            by \code{\link{NRRR.ini}}. Default is NULL. If \code{lang = 'Rcpp'},
#'            then \code{Bg0} is automatically generated by \code{\link{NRRR.ini}}.
#' @param jx number of basis functions to expand the functional predictor.
#' @param jy number of basis functions to expand the functional response.
#' @param p number of predictors.
#' @param d number of responses.
#' @param n sample size.
#' @param maxiter the maximum iteration number of the
#'                blockwise coordinate descent algorithm. Default is 300.
#' @param conv the tolerance level used to control the convergence of the
#'             blockwise coordinate descent algorithm. Default is 1e-4.
# @param quietly a logical value with two options. FALSE (default): show the
#                rank selection process; TRUE: do not show the process.
#' @param method 'RRR' (default): no additional ridge penalty; 'RRS': add an
#'               additional ridge penalty.
#' @param lambda the tuning parameter to control the amount of ridge
#'               penalization. It is only used when \code{method = 'RRS'}.
#'               Default is 0.
# @param ic the user-specified information criterion. Four options are available,
#           including BIC, BICP, AIC, GCV.
#' @param dimred a vector of logical values to decide whether to use cross validation
#'               do rank selection on certain dimensions. TRUE means the rank is selected
#'               by cross validation.
#'               If \code{dimred[1] = FALSE}, r is provided by \code{rankfix}
#'               or \eqn{min(jy*d, rank(X))};
#'               If \code{dimred[2] = FALSE}, rx equals to \code{xrankfix} or p; If \code{dimred[3] = FALSE},
#'               ry equals to \code{yrankfix} or d. Default is \code{c(TRUE, TRUE, TRUE)}.
#' @param rankfix a user-provided value of r when \code{dimred[1] = FALSE}. Default is NULL
#'                which leads to \eqn{r = min(jy*d, rank(X))}.
#' @param xrankfix a user-provided value of rx when \code{dimred[2] = FALSE}. Default is NULL
#'                 which leads to \code{rx = p}.
#' @param yrankfix a user-provided value of ry when \code{dimred[3] = FALSE}. Default is NULL
#'                 which leads to \code{ry = d}.
#' @param lang 'R' (default): the R version function is used; 'Rcpp': the Rcpp
#'             version function is used.
#'
#'
#' @return The function returns a list:
#'   \item{Ag}{the estimated U.}
#'   \item{Bg}{the estimated V.}
#'   \item{Al}{the estimated A.}
#'   \item{Bl}{the estimated B.}
#'   \item{C}{the estimated coefficient matrix C.}
#'   \item{df}{the estimated degrees of freedom of the selected model.}
#'   \item{sse}{the sum of squared errors of the selected model.}
#'   \item{ic}{a vector containing values of BIC, BICP, AIC, GCV of the selected model.}
#'   \item{rx_path}{a matrix displays the path of selecting rx with cross validation.}
#'   \item{ry_path}{a matrix displays the path of selecting ry with cross validation.}
#'   \item{r_path}{a matrix displays the path of selecting r with cross validation.}
#   \item{r_fitseq}{a sequence of possible r values.}
#   \item{iter}{the number of iterations needed to converge in the selected model.}
#'   \item{rank}{the estimated r.}
#'   \item{rx}{the estimated rx.}
#'   \item{ry}{the estimated ry.}
#'
#' @details
#' A three-dimensional grid search procedure of the rank
#' values is performed, and the best model is chosen as the one with the
#' smallest prediction error. Instead of a nested rank selection method, we apply a
#' one-at-a-time selection approach. We first set \eqn{rx = p, ry = d}, and
#' select the best local rank \eqn{\hat r} among the models with
#' \eqn{1 \le r \le min(rank(X), jy*d)}. We then fix the local rank at
#' \eqn{\hat r} and repeat a similar procedure to determine \eqn{\hat rx}
#' and \eqn{\hat ry}, one at a time. Finally, with fixed \eqn{\hat rx} and \eqn{\hat ry},
#' we refine the estimation of r.
#'
#' @references Liu, X., Ma, S., & Chen, K. (2020).
#' Multivariate Functional Regression via Nested Reduced-Rank Regularization.
#' arXiv: Methodology.
#'
# @importFrom rrpack cv.rrr
#' @export
#' @examples
#' library(NRRR)
#' set.seed(1)
#' # Simulation setting 1 in NRRR paper
#' simDat <- NRRR.sim(n = 100, ns = 60, nt = 60, r = 5, rx = 3, ry = 3,
#'                    jx = 8, jy = 8, p = 10, d = 10, s2n = 1,
#'                    rho_X = 0.5, rho_E = 0, Sigma = "CorrAR")
#' # using R function
#' fit_R <- with(simDat, NRRR.cv(Yest, Xest, nfold = 10, norder = NULL,
#'               Ag0 = NULL, Bg0 = NULL, jx = 8, jy = 8, p = 10, d = 10,
#'               n = 100, maxiter = 300, conv = 1e-4,
#'               method = c("RRR", "RRS")[1], lambda = 0,
#'               dimred = c(TRUE, TRUE, TRUE), rankfix = NULL,
#'               xrankfix = NULL, yrankfix = NULL, lang=c('R','Rcpp')[1]))
#' # using Rcpp function
#' fit_Rcpp <- with(simDat, NRRR.cv(Yest, Xest, nfold = 10, norder = NULL,
#'                  Ag0 = NULL, Bg0 = NULL, jx = 8, jy = 8, p = 10,
#'                  d = 10, n = 100, maxiter = 300, conv = 1e-4,
#'                  method = c("RRR", "RRS")[1], lambda = 0,
#'                  dimred = c(TRUE, TRUE, TRUE), rankfix = NULL,
#'                  xrankfix = NULL, yrankfix = NULL, lang=c('R','Rcpp')[2]))
NRRR.cv <- function(Y,X,nfold=10,norder=NULL,Ag0=NULL,Bg0=NULL,jx,jy,p,d,n,
                              maxiter=300,conv=1e-4,#quietly=FALSE,
                              method=c('RRR','RRS')[1],lambda=0,
                              dimred = c(TRUE,TRUE,TRUE),
                              rankfix=NULL,xrankfix=NULL,
                              yrankfix=NULL, lang=c('R','Rcpp')[1]
){
  #require(rrpack)
  if (method == "RRS" & lambda == 0) stop("A positive tuning parameter should be provided when 'RRS' is used.")
  if (!(lang %in% c('R','Rcpp'))) stop("Please choose from 'R' and 'Rcpp'")

  xr <- sum(svd(X)$d>1e-2)
  if (is.null(norder))
    norder <- sample(seq_len(n),n)

  # initialize r
  if(dimred[1]){
    fitRRR <- cv.rrr(Y,X,nfold=10,norder = norder)
    rest <- fitRRR$rank
    # If zero fit
    if(rest==0){
      fitRRR <- RRR(Y,X,nrank=1)
      rest <- fitRRR$rank
    }
    # if(!quietly) {
    #   cat("Initial r   = ",rest, "\n",sep="")
    # }
  }else{
    rest <- ifelse(is.null(rankfix),min(ncol(Y),ncol(X),xr),rankfix)
  }
  rfit <- rest

 if (lang == 'R') {
   # Select rx by cross validation
   rx_path <- matrix(ncol = nfold, nrow = p, NA)
   if (p == 1) {
     rxest <- p
   } else {
     if(dimred[2]){
       ndel <- round(n/nfold)
       for (f in seq_len(nfold)){
         if (f != nfold) {
           iddel <- norder[(1 + ndel * (f - 1)):(ndel * f)]
         }
         else {
           iddel <- norder[(1 + ndel * (f - 1)):n]
         }
         ndel <- length(iddel)
         nf <- n - ndel
         idkeep <- (seq_len(n))[-iddel]
         Xf <- X[-iddel, ]
         Xfdel <- X[iddel, ]
         Yf <- Y[-iddel, ]
         Yfdel <- Y[iddel, ]
         for (i in seq_len(p)){
           rxfit <- i
           ryfit <- d
           fit1 <- NRRR.est(Yf,Xf,NULL,NULL,rini=rfit,rfit,rxfit,ryfit,jx,jy,p,
                            d,n=nf,maxiter=maxiter,conv=conv,#quietly=TRUE,
                            method=method,lambda=lambda)
           rx_path[i,f] <- sum((Yfdel-Xfdel%*%fit1$C)^2)
         }
       }
       index <- order(colSums(rx_path))
       crerr <- rowSums(rx_path[, index])/length(index) * nfold
       rxest <- which.min(crerr)
     } else {
       rxest <- ifelse(is.null(xrankfix),p,xrankfix)
     }
   }

   # if(!quietly) {
   #   cat("Selected rx = ",rxest, "\n",sep="")
   # }

   # Select ry by cross validation
   ry_path <- matrix(ncol = nfold, nrow = d, NA)
   if (d == 1) {
     ryest <- d
   } else {
     if(dimred[3]){
       ndel <- round(n/nfold)
       for (f in seq_len(nfold)){
         if (f != nfold) {
           iddel <- norder[(1 + ndel * (f - 1)):(ndel * f)]
         }
         else {
           iddel <- norder[(1 + ndel * (f - 1)):n]
         }
         ndel <- length(iddel)
         nf <- n - ndel
         idkeep <- (seq_len(n))[-iddel]
         Xf <- X[-iddel, ]
         Xfdel <- X[iddel, ]
         Yf <- Y[-iddel, ]
         Yfdel <- Y[iddel, ]
         for (i in seq_len(d)){
           rxfit <- rxest
           ryfit <- i
           fit1 <- NRRR.est(Yf,Xf,NULL,NULL,rini=rfit,rfit,rxfit,ryfit,jx,jy,p,
                            d,n=nf,maxiter=maxiter,conv=conv,#quietly=TRUE,
                            method=method,lambda=lambda)
           ry_path[i,f] <- sum((Yfdel-Xfdel%*%fit1$C)^2)
         }
       }
       index <- order(colSums(ry_path))
       crerr <- rowSums(ry_path[, index])/length(index) * nfold
       ryest <- which.min(crerr)
     } else {
       ryest <- ifelse(is.null(yrankfix),d,yrankfix)
     }
   }

   # if(!quietly) {
   #   cat("Selected ry = ",ryest, "\n",sep="")
   # }

   # refine rank selection by cross validation
   rfitseq <- max(1,rest-5):min(rest+5,min(n,p*jx,d*jy))
   r_path <- matrix(ncol = nfold, nrow = length(rfitseq), NA)
   if(dimred[1]){
     rxfit <- rxest
     ryfit <- ryest

     ndel <- round(n/nfold)
     for (f in seq_len(nfold)){
       if (f != nfold) {
         iddel <- norder[(1 + ndel * (f - 1)):(ndel * f)]
       }
       else {
         iddel <- norder[(1 + ndel * (f - 1)):n]
       }
       ndel <- length(iddel)
       nf <- n - ndel
       idkeep <- (seq_len(n))[-iddel]
       Xf <- X[-iddel, ]
       Xfdel <- X[iddel, ]
       Yf <- Y[-iddel, ]
       Yfdel <- Y[iddel, ]
       for (i in 1:length(rfitseq)){
         rfit <- rfitseq[i]
         fit1 <- NRRR.est(Yf,Xf,NULL,NULL,rini=rfit,rfit,rxfit,ryfit,jx,jy,p,
                          d,n=nf,maxiter=maxiter,conv=conv,#quietly=TRUE,
                          method=method,lambda=lambda)
         r_path[i,f] <- sum((Yfdel-Xfdel%*%fit1$C)^2)
       }
     }
     index <- order(colSums(r_path))
     crerr <- rowSums(r_path[, index])/length(index) * nfold
     rest <- rfitseq[which.min(crerr)]
     fit <- NRRR.est(Y,X,NULL,NULL,rini=rest,rest,rxfit,ryfit,jx,jy,p,
                     d,n,maxiter=maxiter,conv=conv,#quietly=TRUE,
                     method=method,lambda=lambda)

   } else {
     rxfit <- rxest
     ryfit <- ryest
     fit <- NRRR.est(Y,X,NULL,NULL,rini=rest,rest,rxfit,ryfit,jx,jy,
                     p,d,n,maxiter=maxiter,conv=conv,#quietly=TRUE,
                     method=method,lambda=lambda)

   }
   if (fit$iter == maxiter) stop("The algorithm reaches the maximum iteration.")

   return(list(Ag=fit$Ag,Bg=fit$Bg,Al=fit$Al,Bl=fit$Bl,C=fit$C,df=fit$df,
               sse=fit$sse,ic=fit$ic,#obj=fit$obj,
               rx_path=rx_path,ry_path=ry_path,
               r_path=r_path,#rfitseq=rfitseq,
               #iter=fit$iter,
               rank=rest,rx=rxest,ry=ryest))
 } else {
   method <- ifelse(method == 'RRR', 1, 2)

   dimred1 <- ifelse(dimred[1],1,0)
   dimred2 <- ifelse(dimred[2],1,0)
   dimred3 <- ifelse(dimred[3],1,0)

   xrankfix <- ifelse(is.null(xrankfix),0,xrankfix)
   yrankfix <- ifelse(is.null(yrankfix),0,yrankfix)

   norder <- norder - 1

   fit <- nrrr_cv_my(Y, X, norder, nfold, xr, rfit, xrankfix, yrankfix,
                     jx, jy, p, d, n, maxiter, method,
                     dimred1, dimred2, dimred3, conv, lambda)
   if( sum(fit$rxErrmat)>0 | sum(fit$ryErrmat)>0 | sum(fit$rErrmat)>0 ) stop('Error occurs or the algorithm reaches the maximum iteration')

   return(list(Ag=fit$Ag,Bg=fit$Bg,Al=fit$Al,Bl=fit$Bl,C=fit$C,df=fit$df,
               sse=fit$sse,ic=fit$ic,#obj=fit$obj,
               rx_path=fit$rx_path,ry_path=fit$ry_path,
               r_path=fit$r_path,
               rank=fit$rank,rx=fit$rx,ry=fit$ry
               #,rxErrmat=fit$rxErrmat,ryErrmat=fit$ryErrmat,rErrmat=fit$rErrmat
               ))
 }
}


C <- matrix(c(-2,4,6,2,4,1),2,3)
D <- matrix(c(3,9,7,2,5,1,4,-2,8),3,3)
E <- matrix(c(4,8,1,-1,6,-6,0,4,7),3,3)
# C+D cannot be computed because of different dimensions.

x=D+E
y=D-E
z=C%*%D
p=t(E)
x
y
z
p


softmax <- function (x) {
    m <- max(x)
    exps <- exp(x - m)
    s <- sum(exps)
    return(exps / s)
}
score <- function(input) {
    if ((input[3]) >= (2.45)) {
        var0 <- -0.21995015
    } else {
        var0 <- 0.4302439
    }
    if ((input[3]) >= (2.45)) {
        var1 <- -0.19691855
    } else {
        var1 <- 0.29493433
    }
    if ((input[3]) >= (2.45)) {
        if ((input[4]) >= (1.75)) {
            var2 <- -0.20051816
        } else {
            var2 <- 0.36912444
        }
    } else {
        var2 <- -0.21512198
    }
    if ((input[3]) >= (2.45)) {
        if ((input[3]) >= (4.8500004)) {
            var3 <- -0.14888482
        } else {
            var3 <- 0.2796613
        }
    } else {
        var3 <- -0.19143805
    }
    if ((input[4]) >= (1.6500001)) {
        var4 <- 0.40298507
    } else {
        if ((input[3]) >= (4.95)) {
            var4 <- 0.21724138
        } else {
            var4 <- -0.21974029
        }
    }
    if ((input[3]) >= (4.75)) {
        if ((input[4]) >= (1.75)) {
            var5 <- 0.28692952
        } else {
            var5 <- 0.06272897
        }
    } else {
        if ((input[4]) >= (1.55)) {
            var5 <- 0.009899145
        } else {
            var5 <- -0.19659369
        }
    }
    return(softmax(c((0.5) + ((var0) + (var1)), (0.5) + ((var2) + (var3)), (0.5) + ((var4) + (var5)))))
}


#' @title Aggregate keyword-country data and compute DOI
#'
#' @aliases
#' compute_doi
#' compute_doi.numeric
#' compute_doi.list
#'
#' @description
#' The function computes degree of internationalization (DOI) for object
#' keywords. Degree of internationalization is measured based on the
#' distribution of country search scores.
#'
#' @details
#' The function uses an inverted Gini-coefficient
#' `dplyr::coalesce(1 - ineq::ineq(series, type = "Gini"), 0)`
#' as measure for the degree of internationalization. The more uniform the
#' distribution of search scores across all countries, the higher the inverted
#' Gini-coefficient and the greater the degree of internationalization. In
#' addition to the Gini-coefficient, the package uses inverted Herfindahl index
#' `coalesce(1 - sum((series / sum(series))^2), 0)` and inverted Entropy
#' `dplyr::coalesce(-1 * ineq::ineq(series, parameter = 1, type = "entropy"), 0)`
#' as measures for internationalization.
#'
#' @param control Control batch for which the search score is used. Object
#' of type `numeric`.
#' @param object Object batch for which the keyword-country data
#' is aggregated and DOI is computed.  Object of type `numeric`.
#' @param locations List of locations for which the search score is used.
#' Object of type `character`. Defaults to *"countries"*.
#'
#' @seealso
#' * [example_doi()]
#' * [ineq::ineq()]
#'
#' @return
#' Message that data was aggregated successfully. Data is written to table
#' *data_doi*.
#'
#' @examples
#' \dontrun{
#' compute_doi(
#'   object = 1,
#'   control = 1,
#'   locations = "countries"
#' )
#' compute_doi(
#'   object = as.list(1:5),
#'   control = 1,
#'   locations = "countries"
#' )
#' }
#'
#' @export
#' @rdname compute_doi
#' @importFrom DBI dbWriteTable
#' @importFrom dplyr collect
#' @importFrom dplyr filter
#' @importFrom dplyr mutate
#' @importFrom dplyr select
#' @importFrom glue glue
#' @importFrom purrr map_dbl
#' @importFrom purrr map_lgl
#' @importFrom purrr walk
#' @importFrom rlang .data
#' @importFrom tidyr nest
#' @importFrom tidyr pivot_longer

compute_doi <- function(object, control = 1, locations = "countries") UseMethod("compute_doi", object)

#' @rdname compute_doi
#' @method compute_doi numeric
#' @export

compute_doi.numeric <- function(object, control = 1, locations = "countries") {
  control <- unlist(control)
  .check_length(control, 1)
  .check_length(locations, 1)
  .check_input(locations, "character")
  if (length(object) > 1) {
    compute_doi(control = control, object = as.list(object), locations = locations)
  } else {
    walk(list(control, object), .check_batch)
    if (.test_empty(table = "data_doi", batch_c = control, batch_o = object, locations = locations)) {
      data <- filter(.tbl_score, .data$batch_c == control & .data$batch_o == object)
      data <- collect(data)
      tmp_locations <- pull(collect(filter(.tbl_locations, .data$type == locations)), .data$location)
      data <- filter(data, .data$location %in% tmp_locations & .data$synonym == 0)

      # compute doi measures
      data <- pivot_longer(data, cols = contains("score"), names_to = "type", values_to = "score")
      data <- nest(data, data = c(.data$location, .data$score))
      data <- mutate(data, check = map_lgl(.data$data, ~ !all(is.na(.x$score))))
      out1 <- filter(data, .data$check)
      out1 <- mutate(
        out1,
        gini = map_dbl(.data$data, ~ .compute_gini(series = .x$score)),
        hhi = map_dbl(.data$data, ~ .compute_hhi(series = .x$score)),
        entropy = map_dbl(.data$data, ~ .compute_entropy(series = .x$score))
      )
      out2 <- filter(data, !.data$check)
      out2 <- mutate(
        out2,
        gini = NA,
        hhi = NA,
        entropy = NA
      )
      out <- bind_rows(out1, out2)
      out <- select(
        out,
        .data$date,
        .data$keyword,
        .data$type,
        .data$gini,
        .data$hhi,
        .data$entropy
      )

      # write data
      out <- mutate(
        out,
        batch_c = control,
        batch_o = object,
        locations = locations
      )
      dbWriteTable(conn = globaltrends_db, name = "data_doi", value = out, append = TRUE)
    }
    message(glue("Successfully computed DOI | control: {control} | object: {object} [{object}/{total}]", total = max(.keywords_object$batch)))
  }
}

#' @rdname compute_doi
#' @method compute_doi list
#' @export

compute_doi.list <- function(object, control = 1, locations = "countries") {
  walk(object, compute_doi, control = control, locations = locations)
}

#' @title Compute gini coefficient
#'
#' @rdname hlprs
#' @keywords internal
#' @noRd
#'
#' @importFrom dplyr coalesce

.compute_gini <- function(series) {
  out <- coalesce(1 - ineq::ineq(series, type = "Gini"), 0)
  return(out)
}

#' @title Compute herfindahl hirschman index
#'
#' @rdname hlprs
#' @keywords internal
#' @noRd

.compute_hhi <- function(series) {
  out <- coalesce(1 - sum((series / sum(series))^2), 0)
  return(out)
}

#' @title Compute entropy
#'
#' @rdname hlprs
#' @keywords internal
#' @noRd

.compute_entropy <- function(series) {
  out <- coalesce(-1 * ineq::ineq(series, parameter = 1, type = "entropy"), 0)
  if (out == -Inf) {
    out <- 0
  }
  return(out)
}


#' Fitting bivariate mixed MGL and MGL-EV copula regression models.
#'
#' @description \code{MGL.reg.mixed} is used to fit bivariate MGL and MGL-EV copula regression models for continuous and semi-continuous variables.
#' @param obs two-dimensional matrix for observations.
#' @param U two-dimensional matrix for pseudo copula data with values in \eqn{[0,1]} for (F(y1), F(y2)).
#' @param U_ two-dimensional matrix for pseudo copula data for the data (F(y1), F(y2-1)).
#' @param X design matrix.
#' @param copula 'MGL', 'MGL180', "MGL-EV", "MGL-EV180"
#' @param umin threshold value used in the semi-continuous data.
#' @param f two-dimensional matrix for the density function of marginal distributions.
#' @param initpar Initial values for the parameters to be optimized over.
#' @param hessian Logical. Should a numerically differentiated Hessian matrix be returned?
#' @param ... additional arguments, see \code{\link[stats]{optim}} for more details.
#' @importFrom stats qbeta
#' @importFrom stats optim
#' @return A list containing the following components:
#' * loglike: the value of the estimated maximum of the loglikelihood function.
#' * copula: the name of the fitted copula. "MGL180" and "MGL-EV180" denote the survival MGL and MGL-EV copula respectively.
#' * estimates: the point at which the maximum value of the loglikelihood is obtained.
#' * se: the standard errors of the estimators.
#' * AIC, BIC: the goodness fit of the regression models.
#' * hessian: the hessian at the estimated maximum of the loglikelihood (if requested).
#' @details
#' The estimation method is performed via \code{\link[stats]{optim}} function. Y1 and Y2 are both continuous variables.
#' * Y1: continuous data.
#' * Y2: semi-continuous data where Y2>umin is continuous and Y2<=umin is discrete.
#'
#' copula: "MGL180" and "MGLEV180" denote the survival MGL and survival MGL-EV copula respectively.
#' * For "Gumbel" regression model, the copula parameter \deqn{\delta_i = \exp(X\beta) + 1.}
#' * For "MGL", "MGL180", "MGL-EV", "MGL-EV180" regression model, the copula parameter \deqn{\delta_i = \exp(X\beta),} where \eqn{\beta} is the vector of coefficients to be estimated in the copula regression.
#'
#' @examples
#' library(rMGLReg)
#' u <- cbind(earthqCHI$u1, earthqCHI$u2)
#' u_ <- cbind(earthqCHI$u1, earthqCHI$u2_)
#' y <- cbind(earthqCHI$y1, earthqCHI$y2)
#' f <- cbind(earthqCHI$f1, earthqCHI$f2)
#' obs <- y
#' U <- u
#' U_ <- u_
#' umin <- 20
#' library(splines)
#' X <- ns(earthqCHI$year, knots = quantile(earthqCHI$year, c(0.333, 0.667)), intercept = TRUE)
#' m.MGL180 <- MGL.reg.mixed(obs = y, U = U, U_ = U_, umin = umin, f = f, X = X,
#' copula = "MGL180",
#'                          method = "Nelder-Mead",
#'                         control = list(maxit = 100000),
#'                         initpar = c(0.64, 1.2, 1, -0.2))
#'
#' m.MGLEV180 <- MGL.reg.mixed(obs = y, U = U, U_ = U_, umin = umin, f = f, X = X,
#'                            copula = "MGL-EV180",
#'                            method = "Nelder-Mead",
#'                           control = list(maxit = 100000),
#'                           initpar = c(-0.32, 1, 1, 1))
#' m.MGL180
#' m.MGLEV180
#'
#' @export
#'
#'
MGL.reg.mixed <- function(obs, U, U_, f, X, copula = c(
                            "MGL", "MGL180", "MGL-EV",
                            "MGL-EV180",
                            "Gumbel", 'MGB2'
                          ),
                          umin = 0,
                          hessian = TRUE, initpar, ...) {
  dcMGL.reg <- function(U, param) {
    dim <- length(U)
    a <- 1 / param[1]
    q <- qbeta(1 - U, shape1 = 0.5, shape2 = a) / (1 - qbeta(1 - U, shape1 = 0.5, shape2 = a))
    logdc <- (dim - 1) * lgamma(a) + lgamma(a + dim / 2) - dim * lgamma(a + 0.5) + (a + 0.5) * sum(log(q + 1)) - (a + dim / 2) * log(sum(q) + 1)
    out <- exp(logdc)
    return(out)
  }

  dcMGL180.reg <- function(U, param) {
    dcMGL.reg(1 - U, param = param)
  }
  dcMGLEV180.reg <- function(U, param) {
    u1 <- U[1]
    u2 <- U[2]
    as.numeric(dcMGLEV180.bivar(u1, u2, param = param[1]))
  }

  dcMGLEV.reg <- function(U, param) {
    u1 <- U[1]
    u2 <- U[2]
    as.numeric(dcMGLEV.bivar(u1, u2, param = param[1]))
  }

  dgumcop.reg <- function(U, param) {
    as.numeric(fCopulae::devCopula(u = U[1], v = U[2], type = "gumbel", param = param[1])) # Bivariate Extreme
  }

  hcMGL.reg <- function(U, param) {
    hfunc1 <- hcMGL.bivar(u1 = U[1], u2 = U[2], pars = param[1])$hfunc1
    hfunc2 <- hcMGL.bivar(u1 = U[1], u2 = U[2], pars = param[1])$hfunc2
    out <- list(hfunc1 = hfunc1, hfunc2 = hfunc2)
    out
  }

  hcMGL180.reg <- function(U, param) {
    hfunc1 <- 1 - hcMGL.bivar(u1 = 1 - U[1], u2 = 1 - U[2], pars = param[1])$hfunc1
    hfunc2 <- 1 - hcMGL.bivar(u1 = 1 - U[1], u2 = 1 - U[2], pars = param[1])$hfunc2
    out <- list(hfunc1 = hfunc1, hfunc2 = hfunc2)
    out
  }

  hcMGLEV180.reg <- function(U, param) {
    hfunc1 <- hcMGLEV180.bivar(u1 = U[1], u2 = U[2], param = param[1])$hfunc1
    hfunc2 <- hcMGLEV180.bivar(u1 = U[1], u2 = U[2], param = param[1])$hfunc2
    out <- list(hfunc1 = hfunc1, hfunc2 = hfunc2)
    out
  }
  hcMGLEV.reg <- function(U, param) {
    hfunc1 <- 1 - hcMGLEV180.bivar(u1 = 1 - U[1], u2 = 1 - U[2], param = param[1])$hfunc1
    hfunc2 <- 1 - hcMGLEV180.bivar(u1 = 1 - U[1], u2 = 1 - U[2], param = param[1])$hfunc2
    out <- list(hfunc1 = hfunc1, hfunc2 = hfunc2)
    out
  }

  hgumcop.reg <- function(U, param) {
    VineCopula::BiCopHfunc(u1 = U[1], u2 = U[2], family = 4, par = param[1])
  }

  if (copula == "MGL") {
    dcop <- dcMGL.reg
    hcop <- hcMGL.reg
  } else if (copula == "MGL180") {
    dcop <- dcMGL180.reg
    hcop <- hcMGL180.reg
  } else if (copula == "MGL-EV") {
    dcop <- dcMGLEV.reg
    hcop <- hcMGLEV.reg
  } else if (copula == "MGL-EV180") {
    dcop <- dcMGLEV180.reg
    hcop <- hcMGLEV180.reg
  } else if (copula == "Gumbel") {
    dcop <- dgumcop.reg
    hcop <- hgumcop.reg
  }
  # Obs1 <- obs[, 1] # y1 - the first vector of observations
  Obs2 <- obs[, 2] # y2 - the second vector of observations
  f1 <- f[, 1]
  f2 <- f[, 2]
  copLogL <- function(pars, X) {
    ll <- 0
    if (copula == "Gumbel") {
      delta <- exp(X %*% pars) + 1
      for (i in seq_len(nrow(X))) {
        if (Obs2[i] <= umin) {
          ll[i] <- f1[i] * (hcop(U[i, ], param = delta[i])$hfunc1 - hcop(U_[i, ], param = delta[i])$hfunc1)
        } else {
          ll[i] <- f1[i] * f2[i] * dcop(U[i, ], param = delta[i])
        }
      }
    } else if(copula == "MGB2"){
      p1 <- exp(pars[ncol(X) + 1])
      p2 <- exp(pars[ncol(X) + 2])
      q <- exp(X%*%pars[1:(ncol(X))])
      for (i in seq_len(nrow(X))) {
        if (Obs2[i] <= umin) {
          ll[i] <- f1[i] * (hcMGB2.bivar(u1 = U[i,1], u2 = U[i,2], pars1 = p1, pars2 = p2, pars3 = q[i])$hfunc1 - hcMGB2.bivar(u1 = U[i,1], u2 = U_[i,2], pars1 = p1, pars2 = p2, pars3 = q[i])$hfunc1)
        } else {
          ll[i] <- f1[i] * f2[i] * dcMGB2.bivar(u1 = U[i,1], u2 = U[i,2], pars1 = p1, pars2 = p2, pars3 = q[i])
        }
      }
    } else {
      delta <- exp(X %*% pars)
      for (i in seq_len(nrow(X))) {
        if (Obs2[i] <= umin) {
          ll[i] <- f1[i] * (hcop(U[i, ], param = delta[i])$hfunc1 - hcop(U_[i, ], param = delta[i])$hfunc1)
        } else {
          ll[i] <- f1[i] * f2[i] * dcop(U[i, ], param = delta[i])
        }
      }
    }

    res <- - sum((log(ll)))
    return(res)
  }

  resopt <- optim(
    par = initpar,
    fn = copLogL,
    X = X,
    hessian = hessian, ...
  )
  resopt

  list(
    loglike = -resopt$value,
    copula = list(name = copula),
    estimates = resopt$par,
    se = sqrt(diag(solve(resopt$hessian))),
    hessian = -resopt$hessian,
    AIC = 2 * length(resopt$par) + 2 * resopt$value,
    BIC = log(nrow(U)) * length(resopt$par) + 2 * resopt$value
  )
}


#' @title Fitting bivariate MGL copula models for mixed data
#' @description \code{MGL.mle.mixed} is used to fit bivariate mixed copula regression models via maximum likelihood (ML) method for continuous and semi-continuous variables.
#' @param U two-dimenstional matrix for pseudo copula data with values in \eqn{[0,1]} for (F(y1), F(y2)).
#' @param copula copula 'MGL', 'MGL180', "MGL-EV", "MGL-EV180", "MGB2", "Normal" , "t".
#' @param hessian Logical. Should a numerically differentiated Hessian matrix be returned?
#' @param initpar Initial values for the parameters to be optimized over.
#' @param U_ two-dimensional matrix for pseudo copula data for the data (F(y1), F(y2-1)).
#' @param obs two-dimensional matrix for loss observations (y1, y2).
#' @param f values of the density function for marginal distribution.
#' @param umin threshold value used in the semi-continuous data.
#' @param ... additional arguments, see \code{\link[stats]{nlm}} for more details.
#' @importFrom stats nlm
#' @md
#' @return A list containing the following components:
#' * loglike: the value of the estimated maximum of the loglikelihood function.
#' * copula: the name of the fitted copula. "MGL180" and "MGL-EV180" denote the survival MGL and MGL-EV copula respectively.
#' * estimates: the point at which the maximum value of the loglikelihood is obtained.
#' * se: the standard errors of the estimators.
#' * AIC, BIC: the goodness fit of the regression models.
#' * hessian: the hessian at the estimated maximum of the loglikelihood (if requested).
#'
#' @details
#' The estimation method is performed via \code{\link[stats]{nlm}} function.
#' Y1: continuous variable
#' Y2: semi-continuous variable when Y2>umin, it is continuous and Y2<=umin is discrete.
#'
#' For a portfolio of \eqn{n} observations \eqn{(y_{i1},y_{i2}; \; i=1,\ldots,n)}, the joint density function of \eqn{(Y_1,Y_2)} can be written as
#' \deqn{
#' 	f_{Y_{1},Y_2}(y_{i1},y_{i2})=\begin{cases}
#' 	f_{Y_1}(y_{i1})[
#' 	h_{2|1}(F_{Y_{1}}(y_{i1}),F_{Y_{2}}(y_{i2})) - h_{2|1}(F_{Y_{1}}(y_{i1}),F_{Y_{2}}(y_{i2}-1))
#' 	], & y_{i2}\le umin,\\
#' 	f_{Y_1}(y_{i1})f_{Y_2}(y_{i2})c(F_{Y_{1}}(y_{i1}), F_{Y_{2}}(y_{i2})), & y_{i2} > umin,
#' 	\end{cases}
#' }
#' where the density \eqn{f_{Y_j}(\cdot)} and cdf \eqn{F_{Y_j}(\cdot)} of the marginal distributions  (\eqn{i=1,2}) are specified respectively. Here
#' \eqn{h_{2|1}(u_1, u_2)=\partial C(u_1,u_2)/\partial u_1} is the \eqn{h}-function of bivariate copula.
#'
#'
#' copula:
#' * "MGB2" is multivariate GB2.
#' * "Normal" and "t" denote the Gaussian copula and Student-t copula respectively.
#' * "MGL" and "MGL-EV" denote the MGL and MGL-EV copula respectively.
#' * "MGL180" and "MGL-EV180" denote the survival MGL and survival MGL-EV copula respectively.
#' * "Gumbel" is Gumbel copula.#'
#'
#' @examples
#' library(rMGLReg)
#' # load the Chinese earthquake data set
#' u <- cbind(earthqCHI$u1, earthqCHI$u2) # cdf of marginal distribution
#' u_ <- cbind(earthqCHI$u1, earthqCHI$u2_) # cdf of marginal distribution for Y1 and Y2 - 1
#' y <- cbind(earthqCHI$y1, earthqCHI$y2)  # observations
#' f <- cbind(earthqCHI$f1, earthqCHI$f2) # pdf of marginal distribution
#' obs <- y
#' U <- u
#' U_ <- u_
#' umin <- 20
#' m.MGLMGA180 <- MGL.mle.mixed(obs = y, U = U, U_ = U_,
#' umin = umin, f = f,
#' copula = "MGL180",
#' method = "L-BFGS-B", initpar = c(2))
#' m.MGLMGA180
#' @export
#'
MGL.mle.mixed <- function(obs, U, U_, f, copula = c(
                            "MGL", "MGL180", "MGL-EV",
                            "MGL-EV180",
                            "Gumbel",
                            "Normal", "MGB2", "t"
                          ), umin,
                          hessian = TRUE,
                          initpar, ...) {
  dnormcop <- function(U, param) {
    as.numeric(fCopulae::dellipticalCopula(U, rho = param[1], type = "norm"))
  } # normal copula

  dtcop <- function(U, param) {
    as.numeric(fCopulae::dellipticalCopula(U,
      rho = param[1], type = "t",
      param = param[2]
    ))
  } # t copula

  dgumcop <- function(U, param) {
    as.numeric(fCopulae::devCopula(U, type = "gumbel", param = param[1]))
  } # Bivariate Extreme

  dMGL <- function(U, param) {
    as.numeric(dcMGL.bivar(u1 = U[, 1], u2 = U[, 2], pars = param[1]))
  }

  dMGL180 <- function(U, param) {
    as.numeric(dcMGL180.bivar(u1 = U[, 1], u2 = U[, 2], pars = param[1]))
  }

  dMGB2 <- function(U, param) {
    as.numeric(dcMGB2.bivar(u1 = U[, 1], u2 = U[, 2], pars1 = param[1], pars2 = param[2], pars3 = param[3]))
  }

  dMGLEV <- function(U, param) {
    as.numeric(dcMGLEV.bivar(u1 = U[, 1], u2 = U[, 2], param = param[1])) # Bivariate Extreme
  }

  dMGLEV180 <- function(U, param) {
    as.numeric(dcMGLEV180.bivar(u1 = U[, 1], u2 = U[, 2], param = param[1])) # Bivariate Extreme
  }

  hnormcop <- function(U, param) {
    VineCopula::BiCopHfunc(U[, 1], U[, 2], family = 1, par = param[1])
  }
  htcop <- function(U, param) {
    VineCopula::BiCopHfunc(U[, 1], U[, 2], family = 2, par = param[1], par2 = param[2])
  }
  hgumcop <- function(U, param) {
    VineCopula::BiCopHfunc(U[, 1], U[, 2], family = 4, par = param[1])
  }
  hMGL <- function(U, param) {
    hcMGL.bivar(u1 = U[, 1], u2 = U[, 2], pars = param[1])
  }
  hMGL180 <- function(U, param) {
    hfunc1 <- 1 - hcMGL.bivar(u1 = 1 - U[, 1], u2 = 1 - U[, 2], pars = param[1])$hfunc1
    hfunc2 <- 1 - hcMGL.bivar(u1 = 1 - U[, 1], u2 = 1 - U[, 2], pars = param[1])$hfunc2
    out <- list(hfunc1 = hfunc1, hfunc2 = hfunc2)
    out
  }
  hMGLEV180 <- function(U, param) {
    hfunc1 <- hcMGLEV180.bivar(u1 = U[, 1], u2 = U[, 2], param = param[1])$hfunc1
    hfunc2 <- hcMGLEV180.bivar(u1 = U[, 1], u2 = U[, 2], param = param[1])$hfunc2
    out <- list(hfunc1 = hfunc1, hfunc2 = hfunc2)
    out
  }
  hMGLEV <- function(U, param) {
    hfunc1 <- 1 - hcMGLEV180.bivar(u1 = 1 - U[, 1], u2 = 1 - U[, 2], param = param[1])$hfunc1
    hfunc2 <- 1 - hcMGLEV180.bivar(u1 = 1 - U[, 1], u2 = 1 - U[, 2], param = param[1])$hfunc2
    out <- list(hfunc1 = hfunc1, hfunc2 = hfunc2)
    out
  }
  hMGB2 <- function(U, param) {
    hcMGB2.bivar(u1 = U[, 1], u2 = U[, 2], pars1 = param[1], pars2 = param[2], pars3 = param[3])
  }

  argnorm <- list(length = 1, lower = 0, upper = 1, name = "Gaussian")
  argt <- list(length = 2, lower = c(0, 0), upper = c(1, 100), name = "Student")
  arggum <- list(length = 1, lower = 1, upper = 50, name = "Gumbel")
  argMGB2 <- list(length = 3, lower = c(0, 0, 0), upper = c(50, 50, 50), name = "MGB2")
  argMG180 <- list(length = 1, lower = 0, upper = 10, name = "MGL180")
  argMGLEV180 <- list(length = 1, lower = 0, upper = 10, name = "MGL-EV180")
  argMG <- list(length = 1, lower = 0, upper = 10, name = "MGL")
  argMGLEV <- list(length = 1, lower = 0, upper = 10, name = "MGL-EV")
  if (copula == "MGL") {
    dcop <- dMGL
    arg.cop <- argMG
    hcop <- hMGL
  } else if (copula == "MGL180") {
    dcop <- dMGL180
    arg.cop <- argMG180
    hcop <- hMGL180
  } else if (copula == "MGL-EV") {
    dcop <- dMGLEV
    arg.cop <- argMGLEV
    hcop <- hMGLEV
  } else if (copula == "MGL-EV180") {
    dcop <- dMGLEV180
    arg.cop <- argMGLEV180
    hcop <- hMGLEV180
  } else if (copula == "Gumbel") {
    dcop <- dgumcop
    arg.cop <- arggum
    hcop <- hgumcop
  } else if (copula == "Normal") {
    dcop <- dnormcop
    arg.cop <- argnorm
    hcop <- hnormcop
  } else if (copula == "t") {
    dcop <- dtcop
    arg.cop <- argt
    hcop <- htcop
  } else if (copula == "MGB2") {
    dcop <- dMGB2
    arg.cop <- argMGB2
    hcop <- hMGB2
  }

  # Obs1 <- obs[, 1] # y1
  Obs2 <- obs[, 2] # y2
  f1 <- f[, 1]
  f2 <- f[, 2]
  copLogL <- function(x) {
    if (all(arg.cop$lower < x && arg.cop$upper > x)) {
      index1 <- which(Obs2 <= umin)
      index2 <- which(Obs2 > umin)
      m1 <- f1[index1] * (hcop(U[index1, ], param = x)$hfunc1 - hcop(U_[index1, ], param = x)$hfunc1)
      logL1 <- (log(m1))
      m2 <- f1[index2] * f2[index2] * dcop(U[index2, ], param = x)
      logL2 <- (log(m2))
      ll <- c(logL1, logL2)
      res <- -sum((ll)) # define the loglikelihood
    } else {
      res <- 100000000000000000
    }
    return(res)
  }

  resopt <- nlm(
    f = copLogL,
    p = initpar,
    hessian = hessian
  )

  # list(
  #   loglike = -resopt$minimum,
  #   copula = list(name = arg.cop$name),
  #   estimates = resopt$estimate,
  #   se = sqrt(diag(solve(resopt$hessian))),
  #   hessian = -resopt$hessian,
  #   AIC = 2 * length(resopt$estimate) + 2 * resopt$minimum,
  #   BIC = log(nrow(U)) * length(resopt$estimate) + 2 * resopt$minimum
  # )
  # resopt
  if (hessian == TRUE){
    out <-   list(
      loglike = -resopt$minimum,
      copula = list(name = arg.cop$name),
      estimates = resopt$estimate,
      se = sqrt(diag(solve(resopt$hessian))),
      hessian = -resopt$hessian,
      AIC = 2 * length(resopt$estimate) + 2 * resopt$minimum,
      BIC = log(nrow(U)) * length(resopt$estimate) + 2 * resopt$minimum
    )
  } else {
    out <-   list(
      loglike = -resopt$minimum,
      copula = list(name = arg.cop$name),
      estimates = resopt$estimate,
      AIC = 2 * length(resopt$estimate) + 2 * resopt$minimum,
      BIC = log(nrow(U)) * length(resopt$estimate) + 2 * resopt$minimum
    )
  }
  out
}


# function that returns basic streak statistics given a sequence
streak_stats = function(results_sequence, streak_length){
  n_trials = length(results_sequence)
  n_success = sum(results_sequence)

  run_lengths = rle(results_sequence)$lengths
  if_run_success = rle(results_sequence)$values
  n_runs = length(run_lengths)

  # runs of successes
  longest_run_success = max(run_lengths*if_run_success)
  n_runs_success = sum(if_run_success)

  # successes after streaks of Successes
  run_lengths_adj = run_lengths - streak_length + 1
  run_values_adj_S =
    if_run_success *
    (!((if_run_success == 1) &
         (run_lengths < streak_length)))
  n_after_Sstreak = sum(run_lengths_adj * run_values_adj_S) -
    run_values_adj_S[length(run_values_adj_S)]
  n_success_after_Sstreak = sum((run_lengths - streak_length) * run_values_adj_S)

  # runs of failures
  longest_run_failure = max(run_lengths*(1-if_run_success))
  n_runs_failure = sum(1-if_run_success)

  # successes after streaks of Failures
  run_values_adj_F =
    (1-if_run_success) *
    (!((if_run_success == 0) &
         (run_lengths < streak_length)))
  n_after_Fstreak = sum(run_lengths_adj * run_values_adj_F) -
    run_values_adj_F[length(run_values_adj_F)]
  n_success_after_Fstreak = n_after_Fstreak -
    sum((run_lengths - streak_length) * run_values_adj_F)

  # Proportion of hits on trials after streaks
  phat_after_Sstreak = n_success_after_Sstreak / n_after_Sstreak

  # Difference in proportion of hits (trials after streaks - all other trials)
  phat_after_no_Sstreak = (n_success - n_success_after_Sstreak)/
    (n_trials - n_after_Sstreak)
  phat_Sstreak_vs_others = phat_after_Sstreak - phat_after_no_Sstreak

  # Difference in proportion of hits (trials after hit streaks - trials after miss streaks)
  phat_after_Fstreak = n_success_after_Fstreak / n_after_Fstreak
  phat_Sstreak_vs_Fstreak = phat_after_Sstreak - phat_after_Fstreak

  # Hit streak frequency - proportion of trials that occur after hit streaks
  Sstreak_frequency  = n_after_Sstreak / (n_trials - streak_length)

  # collect the streak stats for the results sequence
  return(data.frame(phat_after_Sstreak,
                    phat_Sstreak_vs_others,
                    phat_Sstreak_vs_Fstreak,
                    Sstreak_frequency,
                    longest_run_success,
                    n_runs
                    #                     n_runs_success,
                    #                     phat_after_Fstreak,
                    #                     n_runs_failure,
                    #                     longest_run_failure,
  ))
}


### fimd7.R
### R code from Van Buuren, S. (2012).
###		Flexible Imputation of Missing Data.
###		CRC/Chapman & Hall, Boca Raton, FL.
### (c) 2012 Stef van Buuren, www.multiple-imputation.com
### Version 1, 22mar2012
### Version 2, 4nov2015 tested with mice 2.23
### Tested with Mac OS X 10.7.3, R2.14-2, mice 2.12

if (packageVersion("mice")<'2.12') stop("This code requires mice 2.12.")

### Section 7.1 Too many columns

library("mice")
library("lattice")
library("foreign")
library("survival")

### Section 7.1.3 Data exploration

data <- leiden85   ## Note: the leiden85 data is not yet avialable in V2.12
if (!is.data.frame(data)) warning("The code for section 7.1/7.2 requires access to the LEIDEN85 data.")

ini <- mice(data, maxit=0)   # recommended
table(ini$nmis)

table(data$beroep1, useNA="always")

v1 <- names(ini$nmis[ini$nmis==0])
outlist1 <- v1[c(1,3:5,7:10,16:47,51:60,62,64:65,69:72)]
length(outlist1)

### Section 7.1.4 Outflux

fx <- fluxplot(data, main=NULL, cex=0.9)

outlist2 <- row.names(fx)[fx$outflux<0.5]
length(outlist2)

data2 <- data[,!names(data) %in% outlist2]
fx2 <- flux(data2)
outlist3 <- row.names(fx2)[fx2$outflux<0.5]

### Section 7.1.5 Logged events

ini$log[1:3,]

outlist4 <- as.character(ini$log[,"out"])

### Section 7.1.6 Quick predictor selection for wide data

outlist <- unique(c(outlist1, outlist2, outlist4))
length(outlist)

data2 <- data[,!names(data) %in% outlist]

inlist <- c("sex","lftanam","rrsyst","rrdiast")
pred <- quickpred(data2, minpuc=0.5, inc=inlist)

table(rowSums(pred))

rowSums(pred[c("rrsyst","rrdiast"),])

names(data2)[pred["rrsyst",]==1]

vname <- "rrsyst"
y <- cbind(data2[vname], r=!is.na(data2[,vname]))
vdata <- data2[,pred[vname,]==1]
round(cor(y=y,x=vdata,use="pair"),2)

### Section 7.1.7 Generating the imputations

### Smart imputation using quickpred - - TIME COMSUMING (30 MINUTES)
imp.qp <- mice(data2, pred=pred, ridge = 0.0001, seed=29725)

### Blind imputation - MUCH MORE TIME CONSUMING (10 HOURS)
### Not recommended
imp <- mice(data, ridge=0.01, seed=32417, maxit=2)

vnames <- c("rrsyst","rrdiast")
cd1 <- complete(imp)[,vnames]
cd2 <- complete(imp.qp)[,vnames]
typ <- factor(rep(c("blind imputation","quickpred"),each=nrow(cd1)))
mis <- ici(data2[,vnames])
mis <- is.na(imp$data$rrsyst)|is.na(imp$data$rrdiast)
cd <- data.frame(typ=typ,mis=mis,rbind(cd1, cd2))
tp72 <- xyplot(jitter(rrdiast,10) ~ jitter(rrsyst,10) | typ, data = cd, groups = mis,
             xlab = "Systolic BP (mmHg)", ylab = "Diastolic BP (mmHg)",
             col=c(mdc(1),mdc(2)), pch=c(1,19),type=c("g","p"),
             strip = strip.custom(bg="grey95"),
             scales = list(alternating=1, tck=c(1,0)))
print(tp72)

### Section 7.1.8 A further improvements: Survival as predictor variable

dat <- cbind(data2, dead = 1 - data2$dwa)
hazard <- nelsonaalen(dat, survda, dead)

### calculate correlations between hazard, t and logt (not in
tmp <- data.frame(hazard, t=data2$survda, logt=log(data2$survda),
                  SBP=data2$rrsyst, DBP=data2$rrdiast)
round(cor(tmp, use="pair"),3)

### Section 7.2 Sensitivity analysis

### Figure 7.3

fit <- survfit(Surv(survda/365, 1-dwa) ~ is.na(rrsyst), data = data2)
plot(fit, lty = 1, lwd=1.5, xlab="Years since intake",
     ylab="K-M Survival probability", las=1,
     col=c(mdc(4),mdc(5)), mark.time=FALSE)
text(4,0.7,"BP measured")
text(2,0.3,"BP missing")

### Section 7.2.3 Generating imputations under the delta-adjustment
delta <- c(0,-5,-10,-15,-20)
post <- imp.qp$post

### undamped sensitivity analysis
### TIME CONSUMING (SEVERAL HOURS)
imp.all.undamped <- vector("list", length(delta))
for (i in 1:length(delta)) {
  d <- delta[i]
  cmd <- paste("imp[[j]][,i] <- imp[[j]][,i] +",d)
  post["rrsyst"] <- cmd
  imp <- mice(data2, pred=pred, post=post, maxit=1, seed=i*22, ridge=0.0001)
  imp.all.undamped[[i]] <- imp
}

### damped sensitivity analysis
### TIME CONSUMING (SEVERAL HOURS)
imp.all.damped <- vector("list", length(delta))
for (i in 1:length(delta)) {
  d <- delta[i]
  cmd <- paste("fit <- lm(y ~ as.matrix(x));
              damp <- sqrt(1 - summary(fit)$r.squared);
              imp[[j]][, i] <- imp[[j]][, i] + damp * ", d)
  post["rrsyst"] <- cmd
  imp <- mice(data2, pred=pred, post=post, maxit=1, seed=i*22, ridge=0.0001)
  imp.all.damped[[i]] <- imp
}

### Section 7.2.4 Complete data analysis

cda <- expression(
    sbpgp <- cut(rrsyst, breaks = c(50, 124,
          144, 164, 184, 200, 500)),
    agegp <- cut(lftanam, breaks = c(85, 90,
          95, 110)),
    dead  <- 1 - dwa,
    coxph(Surv(survda, dead)
          ~ C(sbpgp, contr.treatment(6, base = 3))
          + strata(sexe, agegp)))
imp <- imp.all.damped[[1]]
fit <- with(imp, cda)
as.vector(exp(summary(pool(fit))[,1]))

### Table 7.5

fit1 <- with(imp.all.damped[[1]], cda)
fit2 <- with(imp.all.damped[[2]], cda)
fit3 <- with(imp.all.damped[[3]], cda)
fit4 <- with(imp.all.damped[[4]], cda)
fit5 <- with(imp.all.damped[[5]], cda)
r1<-as.vector(t(exp(summary(pool(fit1))[,c(1,6:7)])))
r2<-as.vector(t(exp(summary(pool(fit2))[,c(1,6:7)])))
r3<-as.vector(t(exp(summary(pool(fit3))[,c(1,6:7)])))
r4<-as.vector(t(exp(summary(pool(fit4))[,c(1,6:7)])))
r5<-as.vector(t(exp(summary(pool(fit5))[,c(1,6:7)])))
round(t(matrix(c(r1,r2,r3,r4,r5),nrow=15)),2)

### Section 7.3 Correct prevalence estimates from self-reported data

library("mice")
krul <- selfreport[selfreport$src=="krul",]
mgg <- selfreport[selfreport$src=="mgg",]

### Figure 7.4

xy <- xy.coords(krul$bm, krul$br-krul$bm)
plot(xy,col=mdc(1),xlab="Measured BMI",ylab="Reported - Measured BMI",
     xlim=c(17,45),ylim=c(-5,5), type="n",lwd=0.7)
polygon(x=c(30,20,30),y=c(0,10,10),col="grey95",border=NA)
polygon(x=c(30,40,30),y=c(0,-10,-10),col="grey95",border=NA)
abline(0,0,lty=2,lwd=0.7)
points(xy, col=mdc(1),cex=0.7)
lines(lowess(xy),lwd=2,col=mdc(4))
text(1:4,x=c(40,28,20,32),y=c(4,4,-4,-4),cex=3)
box(lwd=1)

### Figure 7.5 Krul data - males
males <- krul[krul$sex=="Male",]
fit <- lm(bm~br,data=males)
plot(x=males$br,y=males$bm,
     xlim=c(26,34),ylim=c(26,34),
     xlab="Self-reported BMI",ylab="Measured BMI",col=mdc(1),
     cex.lab=1.5, cex.axis=1.3)
abline(h=30,v=30,lty=2)
abline(coef(fit),col=mdc(4))
abline(v=(30-coef(fit)[1])/coef(fit)[2],col=mdc(4))
text(1:4,x=c(33.8,33.8,26.2,26.2),y=c(33.8,26.2,26.2,33.8),cex=4)
text(c("a","b"),x=c(29.1,29.7,29.1,29.7),y=c(34.1,34.1,26.4,26.4),cex=3,adj=c(0.5,1))

### Section 7.3.4 Data

md.pattern(selfreport[,c("age","sex","hm","hr","wm","wr")])

### Section 7.3.5 Application

bmi <- function(h,w){return(w/(h/100)^2)}
init <- mice(selfreport,maxit=0)
meth <- init$meth
meth["bm"] <- "~bmi(hm,wm)"
meth[c("prg", "edu", "etn")] <- ""
pred <- init$pred
pred[,c("src","id","pop","prg","edu","etn","web","bm","br")] <- 0
imp <- mice(selfreport, pred=pred, meth=meth, seed=66573, maxit=20, m=10)

### Figure 7.6

cd <- complete(imp, 1)
xy <- xy.coords(cd$bm, cd$br-cd$bm)
plot(xy,col=mdc(2),xlab="Measured BMI",ylab="Reported - Measured BMI",
     xlim=c(17,45),ylim=c(-5,5), type="n",lwd=0.7)
polygon(x=c(30,20,30),y=c(0,10,10),col="grey95",border=NA)
polygon(x=c(30,40,30),y=c(0,-10,-10),col="grey95",border=NA)
abline(0,0,lty=2,lwd=0.7)

idx <- cd$src=="krul"
xyc <- xy; xyc$x <- xy$x[idx]; xyc$y <- xy$y[idx]
xys <- xy; xys$x <- xy$x[!idx]; xys$y <- xy$y[!idx]
points(xyc,col=mdc(1), cex=0.7)
points(xys,col=mdc(2), cex=0.7)
lines(lowess(xyc),col=mdc(4),lwd=2)
lines(lowess(xys),col=mdc(5),lwd=2)
text(1:4,x=c(40,28,20,32),y=c(4,4,-4,-4),cex=3)
box(lwd=1)

### Table 7.7
### Not particularly elegant code, at the moment
  prev <- matrix(NA,nr=15,nc=4)
  prev[1,1:2] <- summary(pool(with(imp, lm(br>=30~1,subset=(src=="mgg")))))[1,1:2]
  prev[1,3:4] <- summary(pool(with(imp, lm(bm>=30~1,subset=(src=="mgg")))))[1,1:2]
  prev[2,1:2] <- summary(pool(with(imp, lm(br>=30~1,subset=(src=="mgg"&sex=="Male")))))[1,1:2]
  prev[2,3:4] <- summary(pool(with(imp, lm(bm>=30~1,subset=(src=="mgg"&sex=="Male")))))[1,1:2]
  prev[3,1:2] <- summary(pool(with(imp, lm(br>=30~1,subset=(src=="mgg"&sex=="Female")))))[1,1:2]
  prev[3,3:4] <- summary(pool(with(imp, lm(bm>=30~1,subset=(src=="mgg"&sex=="Female")))))[1,1:2]
  prev[4,1:2] <- summary(pool(with(imp, lm(br>=30~1,subset=(src=="mgg"&sex=="Male"&age>=18&age<30)))))[1,1:2]
  prev[4,3:4] <- summary(pool(with(imp, lm(bm>=30~1,subset=(src=="mgg"&sex=="Male"&age>18&age<30)))))[1,1:2]
  prev[5,1:2] <- summary(pool(with(imp, lm(br>=30~1,subset=(src=="mgg"&sex=="Male"&age>=30&age<40)))))[1,1:2]
  prev[5,3:4] <- summary(pool(with(imp, lm(bm>=30~1,subset=(src=="mgg"&sex=="Male"&age>30&age<40)))))[1,1:2]
  prev[6,1:2] <- summary(pool(with(imp, lm(br>=30~1,subset=(src=="mgg"&sex=="Male"&age>=40&age<50)))))[1,1:2]
  prev[6,3:4] <- summary(pool(with(imp, lm(bm>=30~1,subset=(src=="mgg"&sex=="Male"&age>40&age<50)))))[1,1:2]
  prev[7,1:2] <- summary(pool(with(imp, lm(br>=30~1,subset=(src=="mgg"&sex=="Male"&age>=50&age<60)))))[1,1:2]
  prev[7,3:4] <- summary(pool(with(imp, lm(bm>=30~1,subset=(src=="mgg"&sex=="Male"&age>50&age<60)))))[1,1:2]
  prev[8,1:2] <- summary(pool(with(imp, lm(br>=30~1,subset=(src=="mgg"&sex=="Male"&age>=60&age<80)))))[1,1:2]
  prev[8,3:4] <- summary(pool(with(imp, lm(bm>=30~1,subset=(src=="mgg"&sex=="Male"&age>60&age<80)))))[1,1:2]
  prev[10,1:2] <- summary(pool(with(imp, lm(br>=30~1,subset=(src=="mgg"&sex=="Female"&age>=18&age<30)))))[1,1:2]
  prev[10,3:4] <- summary(pool(with(imp, lm(bm>=30~1,subset=(src=="mgg"&sex=="Female"&age>18&age<30)))))[1,1:2]
  prev[11,1:2] <- summary(pool(with(imp, lm(br>=30~1,subset=(src=="mgg"&sex=="Female"&age>=30&age<40)))))[1,1:2]
  prev[11,3:4] <- summary(pool(with(imp, lm(bm>=30~1,subset=(src=="mgg"&sex=="Female"&age>30&age<40)))))[1,1:2]
  prev[12,1:2] <- summary(pool(with(imp, lm(br>=30~1,subset=(src=="mgg"&sex=="Female"&age>=40&age<50)))))[1,1:2]
  prev[12,3:4] <- summary(pool(with(imp, lm(bm>=30~1,subset=(src=="mgg"&sex=="Female"&age>40&age<50)))))[1,1:2]
  prev[13,1:2] <- summary(pool(with(imp, lm(br>=30~1,subset=(src=="mgg"&sex=="Female"&age>=50&age<60)))))[1,1:2]
  prev[13,3:4] <- summary(pool(with(imp, lm(bm>=30~1,subset=(src=="mgg"&sex=="Female"&age>50&age<60)))))[1,1:2]
  prev[14,1:2] <- summary(pool(with(imp, lm(br>=30~1,subset=(src=="mgg"&sex=="Female"&age>=60&age<80)))))[1,1:2]
  prev[14,3:4] <- summary(pool(with(imp, lm(bm>=30~1,subset=(src=="mgg"&sex=="Female"&age>60&age<80)))))[1,1:2]

### find group sizes
table(mgg$sex,mgg$web)
table(mgg$sex,mgg$web,mgg$age>55)

### Section 7.4 Enhancing comparability

library("mice")

### Section 7.4.3 Independence: Imputation without a bridge study

fA <- c(242, 43, 15, 0, 6)
fB <- c(145, 110, 29, 8)
YA <- rep(ordered(c(0:3,NA)), fA)
YB <- rep(ordered(c(0:3)), fB)
Y <- rbind(data.frame(YA,YB=ordered(NA)),
           data.frame(YB,YA=ordered(NA)))

md.pattern(Y)

### simulation function
### Warning: micemill() overwrites objects 'imp' and 'tau'
###          in the global enviroment

micemill <- function(n) {
  for (i in 1:n) {
    imp <<- mice.mids(imp) # global assignment
    cors <- with(imp, cor(as.numeric(YA),
                         as.numeric(YB),
                         method="kendall"))
    tau <<- rbind(tau, getfit(cors, s=TRUE))  # global assignment
  }
}

tau <- NULL
imp <- mice(Y, max=0, m=10, seed=32662)
micemill(50)

### Figure 7.7
plotit <- function()
  matplot(x=1:nrow(tau),y=tau,
        ylab=expression(paste("Kendall's ",tau)),
          xlab="Iteration", type="l", lwd=1,
        lty=1:10,col="black")
plotit()

### Section 7.4.4 Fully dependent or independent?
### Use the 'walking' dataset of the mice package

## split walking: ab is sources A and B (598 records)
ab <- walking[walking$src=="A"|walking$src=="B",]

## split walking: euridiss is external source (292 records)
external <- walking[walking$src=="E",]

### create contingency tables: YA by YB (in ab)
ftable(addmargins(table(ab[,c("YA","YB")],useNA="ifany")))

### Table 7.8 contingency table YA by YB (in euridiss)
ftable(addmargins(table(external[,c("YA","YB")],useNA="ifany")))

### Section 7.4.5 Imputation using a bridge study
md.pattern(walking)

### Figure 7.8
tau <- NULL
imp <- mice(walking, max=0, m=10, seed=92786)
pred <- imp$pred
pred[,c("src","age","sex")] <- 0
imp <- mice(walking, max=0, m=10, seed=92786, pred=pred)
micemill(20)
plotit()

### impute without and with covariates

thetaAB <- NULL
props <- with(imp, mean(YB[src=="A"]=='0'))
thetaAB <<- rbind(thetaAB, getfit(props, s=TRUE))

micemill <- function(n) {
  for (i in 1:n) {
    imp <<- mice.mids(imp) # global assignment
    cors <- with(imp, cor(as.numeric(YA[src=="A"]),
                         as.numeric(YB[src=="A"]),
                         method="kendall"))
    tau <<- rbind(tau, getfit(cors, s=TRUE))  # global assignment
    means <- with(imp, mean(as.numeric(YA[src=="A"]), na.rm=TRUE))
    thetaBA <<- rbind(thetaBA, getfit(means, s=TRUE)-1)
    props <- with(imp, mean(YB[src=="A"]=='0'))
    thetaAB <<- rbind(thetaAB, getfit(props, s=TRUE))
    tabs <- with(imp, ftable(addmargins(
                      table(YA[src=="A"],YB[src=="A"],
                            useNA="ifany", dnn=c("YA","YB")))))
    print(getfit(tabs)[[2]])
  }
}

tau <- NULL
thetaBA <- NULL
thetaAB <- NULL
imp <- mice(walking, max=0, m=10, seed=99786)
oldpred <- pred <- imp$pred
pred[,c("src","age","sex")] <- 0
imp <- mice(walking, max=0, m=10, seed=99786, pred=pred)
micemill(20)
pred <- oldpred
pred[,c("src")] <- 0
imp <- mice(walking, max=0, m=10, pred=pred)
micemill(20)

### Figure 7.9

matplot(x=1:nrow(thetaAB),y=thetaAB,xlab="Iteration",
        ylab = expression(hat(theta)[AB]),
        type="l", lwd=1, lty=1:10,col="black",
        ylim=c(0.4,0.7))
abline(h=0.497, v=20, lty=2)
text(x=5,y=0.488,expression(hat(theta)[BB]),adj=0)
text(x=5,y=0.430,"Without covariates", adj=0)
text(x=25,y=0.430,"With covariates", adj=0)
arrows(x0=4.5,y0=0.488,x1=0,y1=0.495,length=0.1, angle=20)
points(x=-0.4,y=0.497,pch=20)

## file <- file.path("~/Documents/Sync/Impute/mice/V2.12/mice/data/walking.rda")
## save(walking, file=file)

## selfreport <- nl
## file <- file.path("~/Documents/Sync/Impute/mice/V2.12/mice/data/selfreport.rda")
## save(selfreport, file=file)


### calculate the sample size needed for achieving a certain power in three ways: TSLS, AR, ARsens
### some parameter values are inferred from the ivmodel

IVsize=function(ivmodel, power, alpha=0.05, beta=NULL, type="TSLS", deltarange=NULL, delta=NULL){
  esti = para(ivmodel)
  if(is.null(beta))
    beta = esti$beta
  if(is.null(deltarange))
    deltarange = ivmodel$deltarange

  if(type=="TSLS"){
     return(TSLS.size(power, beta, cor(ivmodel$Zadj, ivmodel$Dadj), esti$sigmau, var(ivmodel$Dadj), alpha))
  }
  if(type=="AR"){
    return(AR.size(power, ivmodel$p, ivmodel$L, beta, esti$gamma, var(ivmodel$Zadj), esti$sigmau, esti$sigmav, esti$rho, alpha))
  }

  if(type=="ARsens"){
    return(ARsens.size(power, ivmodel$p, beta, esti$gamma, var(ivmodel$Zadj), esti$sigmau, esti$sigmav, esti$rho, alpha, deltarange, delta))
  }
  print("Input error.")
  return(NULL)
}


#install.packages("ggplot2")
require("ggplot2")
library("markovchain")

setwd("C:/Code/climData/glaser2019")

dfHhi <- read.csv("csv/hhi_1500_2019.csv", sep=",", na = "NA")

pic <- dfHhi$hhi
len <- length(pic)
## mirror pic
pic <- append(pic, pic)
for(i in 1:len) {
  pic[i+len] <- pic[1+len-i]
}

frq <- fft(pic, inverse = FALSE)

frq1 <- frq
filterYears = 0.1   #filter 1m
start = round(len/(12.0*filterYears))
stop  = round(2.0*len-start)
frq1[start:stop] <- 0.0
pic1 <- Re(fft(frq1, inverse = TRUE)/length(frq1))
pic1 <- pic1[1:len]
dfHhi$hhi1 <-  pic1

frq5 <- frq
filterYears = 5   #filter 5y
start = round(len/(12.0*filterYears))
stop  = round(2.0*len-start)
frq5[start:stop] <- 0.0
pic5 <- Re(fft(frq5, inverse = TRUE)/length(frq5))
pic5 <- pic5[1:len]
dfHhi$hhi5 <-  pic5

dfHhi$rhi <- dfHhi$hhi
dfHhi$rmhi <- dfHhi$hhi
for(i in 1:length(dfHhi$rhi)) {
  dfHhi$rhi[i] = 'd4'
  if (dfHhi$hhi1[i] > -3.0) {dfHhi$rhi[i] = 'd3'}
  if (dfHhi$hhi1[i] > -2.0) {dfHhi$rhi[i] = 'd2'}
  if (dfHhi$hhi1[i] > -1.5) {dfHhi$rhi[i] = 'd1'}
  if (dfHhi$hhi1[i] > -1.0) {dfHhi$rhi[i] = 'n0'}
  if (dfHhi$hhi1[i] >  1.0) {dfHhi$rhi[i] = 'w1'}
  if (dfHhi$hhi1[i] >  1.5) {dfHhi$rhi[i] = 'w2'}
  if (dfHhi$hhi1[i] >  2.0) {dfHhi$rhi[i] = 'w3'}
  if (dfHhi$hhi1[i] >  3.0) {dfHhi$rhi[i] = 'w4'}
  dfHhi$rmhi[i] <- paste('m',dfHhi$month[i],':',dfHhi$rhi[i], sep='')
}
dfHhi$hstate  <- dfHhi$hhi
dfHhi$hstate[1] <- paste('n0','-',dfHhi$rhi[1], sep='')
dfHhi$mhstate <- dfHhi$hhi
dfHhi$mhstate[1] <- paste('m',dfHhi$month[12],':','n0','-','m',dfHhi$month[1],':',dfHhi$rhi[1], sep='')
for(i in 2:length(dfHhi$hstate)) {
  dfHhi$hstate[i] <- paste(dfHhi$rhi[i-1],'-',dfHhi$rhi[i], sep='')
  dfHhi$mhstate[i] <- paste('m',dfHhi$month[i-1],':',dfHhi$rhi[i-1],'-','m',dfHhi$month[i],':',dfHhi$rhi[i], sep='')
}
monthStates <- c("m1","m2","m3","m4","m5","m6","m7","m8","m9","m10","m11","m12")
humidityStates <- c("w4", "w3", "w2", "w1", "n0", "d1", "d2", "d3", "d4")
byRow <- TRUE
humidityMatrix  <- matrix(rep( 0.0, len=length(humidityStates)^2), nrow=length(humidityStates))
for(row in 1:length(humidityStates)) {
  sumRow <- 0.0
  for(col in 1:length(humidityStates)) {
    trans <- paste(humidityStates[row],'-',humidityStates[col], sep='')
    count <- length(subset(dfHhi, dfHhi$hstate==trans)$hstate)
    humidityMatrix[row,col] <- count
    sumRow <- sumRow + count
  }
  if(sumRow > 0.0)  {
    for(col in 1:length(humidityStates)) {
      humidityMatrix[row,col] <- humidityMatrix[row,col] / sumRow
    }
  }
}

mcHumidity <- new("markovchain", states = humidityStates, byrow = byRow,
                  transitionMatrix = humidityMatrix, name = "Humidity")

initialState <- c(0, 0, 0, 0, 1, 0, 0, 0, 0)
after1000month <- initialState * (mcHumidity ^ 1000)
transitionProbability(mcHumidity, "w4", "w3")

print(mcHumidity)
show(mcHumidity)
steadyStates(mcHumidity) #endless time

simHumidity <- rmarkovchain(n = 12*10, object = mcHumidity, t0 = "n0")

sequence <- c("a", "b", "a", "a", "a", "a", "b", "a", "b", "a", "b", "a", "a",
              "b", "b", "b", "a")
sequenceMatr <- createSequenceMatrix(dfHhi$rhi, sanitize = FALSE)

mcFitMLE <- markovchainFit(data = dfHhi$rhi)
mcFitBSP <- markovchainFit(data = dfHhi$rhi, method = "bootstrap", nboot = 5, name = "Bootstrap Mc")

sequenceMatr <- createSequenceMatrix(dfHhi$rmhi, sanitize = FALSE)
mcFitMLE <- markovchainFit(data = dfHhi$rmhi)
mcFitBSP <- markovchainFit(data = dfHhi$rmhi, method = "bootstrap", nboot = 5, name = "Bootstrap Mc")

steadyStates(mcFitBSP$estimate)
simHumidity <- rmarkovchain(n = 12*10, object = mcFitBSP$estimate, t0 = "m1:n0")

verifyMarkovProperty(dfHhi$rmhi)
verifyEmpiricalToTheoretical(data=dfHhi$rhi,object=mcFitBSP$estimate)
verifyHomogeneity(inputList=list(dfHhi$rmhi[3601:6000], dfHhi$rmhi[1:2400]),verbose=TRUE)

## create monthly
mcFitBSP <- markovchainFit(data = dfHhi$rmhi, method = "bootstrap", nboot = 5, name = "Bootstrap Mc")

from = "d2"
to = "d1"

props <- monthStates
mon <- monthStates
for(i in 1:length(monthStates)) {
  start = paste(monthStates[i],':',from, sep='')
  stop  = paste(monthStates[(i+1)%%length(monthStates)],':',to, sep='')
  props[i] = transitionProbability(mcFitBSP$estimate, start, stop)
  mon[i] <- 0.01*i
}

mp <- ggplot()
mp +
  theme_classic() +
  geom_point(aes(y=props, x=mon))

## 100 years

from = "w1"
to = "w2"

result <- data.frame(years = c(1:9))
mp <- ggplot(result) + theme_classic() +
  scale_y_continuous(limits=c(0,1))+
  scale_x_continuous(limits=c(1500,2020))

for (from in humidityStates) {
  for (to in humidityStates) {
  result$prop <- c(1:9)
for (i in 1:9) {
  yS = 1450 + 50*i
  yE = 1550 + 50*i
  result$years[i] <- 1500 + 50*i
  seq <- subset(dfHhi, dfHhi$year>yS & dfHhi$year<yE)
  mcFitBSP <- markovchainFit(data = seq$rhi, method = "bootstrap", nboot = 5, name = "Bootstrap Mc")
  steadyStates(mcFitBSP$estimate)
  result$prop[i] <-transitionProbability(mcFitBSP$estimate, from, to)

}
  newName <- paste(from,'-',to, sep='')
  names(result)[names(result) == 'prop'] <- newName
  mp <- mp + geom_line(aes(y=!!result[,newName], x=result[,"years"]))

  }
}

### only look for rsults

result <- data.frame(years = c(1:9))
for (i in 1:9) {
  yS = 1450 + 50*i
  yE = 1550 + 50*i
  result$years[i] <- 1500 + 50*i
  seq <- subset(dfHhi, dfHhi$year>yS & dfHhi$year<yE)
  mcFitBSP <- markovchainFit(data = seq$rhi, method = "bootstrap", nboot = 5, name = "Bootstrap Mc")
  states <- steadyStates(mcFitBSP$estimate)
  for(col in colnames(states)){
     result[i,col] <- states[,col]
  }
}

## continous
result <- data.frame(years = c(1600:2018))
for (yE in 1600:2018) {
  yS = yE - 100
  result$years[yE-1599] <- yE - 50
  seq <- subset(dfHhi, dfHhi$year>yS & dfHhi$year<yE)
  mcFitBSP <- markovchainFit(data = seq$rhi, method = "bootstrap", nboot = 5, name = "Bootstrap Mc")
  states <- steadyStates(mcFitBSP$estimate)
  for(col in colnames(states)){
    result[yE-1599,col] <- states[,col]
  }
}

mp <- ggplot(result) + theme_classic() +
  scale_y_continuous(limits=c(0,1))+
  scale_x_continuous(limits=c(1500,2020)) +
  geom_line(aes(y=result$d4, x=result$years, col="iD4")) +
  geom_line(aes(y=result$d3, x=result$years, col="hD3")) +
  geom_line(aes(y=result$d2, x=result$years, col="gD2")) +
  geom_line(aes(y=result$d1, x=result$years, col="fD1")) +
  geom_line(aes(y=result$n0, x=result$years, col="eN0")) +
  geom_line(aes(y=result$w1, x=result$years, col="dW1")) +
  geom_line(aes(y=result$w2, x=result$years, col="cW2")) +
  geom_line(aes(y=result$w3, x=result$years, col="bW3")) +
  geom_line(aes(y=result$w4, x=result$years, col="aW4"))
mp

result$wavg <- (result$w4*4 + result$w3*3 + result$w2*2 + result$w1*1)
result$davg <- 0-(result$d4*4 + result$d3*3 + result$d2*2 + result$d1*1)
result$avg <- (result$davg + result$wavg)/2

mp <- ggplot(result) + theme_classic() +
  scale_y_continuous(limits=c(-1.5,1.5))+
  scale_x_continuous(limits=c(1500,2020)) +
  geom_line(aes(y=result$wavg, x=result$years, col="wavg")) +
  geom_line(aes(y=result$avg, x=result$years, col="avg")) +
  geom_line(aes(y=result$davg, x=result$years, col="davg"))
mp

mp <- ggplot(dfHhi, aes(year, month))
mp + geom_raster(aes(fill=hhi1))+
  #theme_classic(base_size=80) +
  theme_classic() +
  scale_y_continuous(breaks=c(1,6,12))+
  scale_x_continuous(limits=c(1500,2020)) +
  scale_fill_gradient2(low="#AA6010", mid="#FCF0C2", high="#23AB30") +
  theme( legend.key.width = unit(2,"cm")) +
  geom_line(aes(y=6+2*dfHhi$hhi5, x=dfHhi$time)) +
  #geom_line(aes(y=10+0.5*dfHhi$hhi, x=dfHhi$time)) +
  guides(fill=guide_legend(title="PI", reverse = TRUE))

library("psych")
pairs.panels(result)

cr <- cor(result)


\name{only_F_estimate}
\alias{only_F_estimate}
%- Also NEED an '\alias' for EACH other topic documented here.
\title{
  Estimating node fitnesses in isolation
}
\description{
  This function estimates node fitnesses \eqn{\eta_i} assusming either \eqn{A_k = k} (i.e. linear preferential attachment) or \eqn{A_k = 1} (i.e. no preferential attachment). The method has a hyper-parameter \eqn{s}. It first performs a cross-validation to select the optimal parameter \eqn{s} for the prior of \eqn{\eta_i}, then estimates \eqn{eta_i} with the full data (Ref. 1).
}
\usage{
only_F_estimate(net_object                             ,
               net_stat    = get_statistics(net_object),
               p           = 0.75                      ,
               stop_cond   = 10^-8                     ,
               model_A     = "Linear"                  ,
               ...)
}
%- maybe also 'usage' for other objects documented here.
\arguments{
 \item{net_object}{
    an object of class \code{PAFit_net} that contains the network.
  }
\item{net_stat}{
   An object of class \code{PAFit_data} which contains summerized statistics needed in estimation. This object is created by the function \code{\link{get_statistics}}. The default value is \code{get_statistics(net_object)}.
}
  \item{p}{Numeric. This is the ratio of the number of new edges in the learning data to that of the full data. The data is then divided into two parts: learning data and testing data based on \code{p}. The learning data is used to learn the node fitnesses and the testing data is then used in cross-validation. Default value is \code{0.75}.}
\item{stop_cond}{Numeric. The iterative algorithm stops when \eqn{abs(h(ii) - h(ii + 1)) / (abs(h(ii)) + 1) < stop.cond} where \eqn{h(ii)} is the value of the objective function at iteration \eqn{ii}. We recommend to choose \code{stop.cond} at most equal to \eqn{10^(- number of digits of h - 2)}, in order to ensure that when the algorithm stops, the increase in posterior probability is less than 1\% of the current posterior probability. Default is \code{10^-8}. This threshold is good enough for most applications.}

  \item{model_A}{String. Indicates which attachment function \eqn{A_k} we assume:
      \itemize{
        \item \code{"Linear"}: We assume \eqn{A_k = k}, i.e. the Bianconi-\enc{Barabási}{Barabasi} model (Ref. 2).
        \item \code{"Constant"}: We assume \eqn{A_k = 1}, i.e. the Caldarelli model (Ref. 3).
      }
  }
    \item{\dots}{
    %%     ~~Describe \code{\dots} here~~
  }
}

\value{
   Outputs a \code{Full_PAFit_result} object, which is a list containing the following fields:
  \itemize{
    \item \code{cv_data}: a \code{CV_Data} object which contains the cross-validation data. Normally the user does not need to pay attention to this data.

    \item \code{cv_result}: a \code{CV_Result} object which contains the cross-validation result. Normally the user does not need to pay attention to this data.

    \item \code{estimate_result}: this is a \code{PAFit_result} object which contains the estimated node fitnesses and their confidence intervals. In particular, the important fields are:
    \itemize{
    \item \code{shape}: this is the selected value for the hyper-parameter \eqn{s}.
    \item \code{g}: the number of bins used.
    \item \code{f}: the estimated node fitnesses.
    \item \code{var_f}: the estimated variance of \eqn{\eta_i}.
    \item \code{upper_f}: the estimated upper value of the interval of two standard deviations around \eqn{\eta_i}.
    \item \code{lower_f}: the estimated lower value of the interval of two standard deviations around \eqn{\eta_i}.
    \item \code{objective_value}: values of the objective function over iterations in the final run with the full data.
    \item \code{diverge_zero}: logical value indicates whether the algorithm diverged in the final run with the full data.
}
  }
}
\author{
  Thong Pham \email{thongpham@thongpham.net}
}
\references{
  1. Pham, T., Sheridan, P. & Shimodaira, H. (2016). Joint Estimation of Preferential Attachment and Node Fitness in Growing Complex Networks. Scientific Reports 6, Article number: 32558. doi:10.1038/srep32558   (\url{http://www.nature.com/articles/srep32558}).

  2. Bianconni, G. & \enc{Barabási}{Barabasi}, A. (2001). Competition and multiscaling in evolving networks. Europhys. Lett., 54, 436 (\url{http://iopscience.iop.org/article/10.1209/epl/i2001-00260-6/meta}).

  3. Caldarelli, G., Capocci, A. , De Los Rios, P. & \enc{Muñoz}{Munoz}, M.A. (2002). Scale-Free Networks from Varying Vertex Intrinsic Fitness. Phys. Rev. Lett., 89, 258702 (\url{http://link.aps.org/doi/10.1103/PhysRevLett.89.258702}).

}
\seealso{
  See \code{\link{get_statistics}} for how to create summerized statistics needed in this function.

  See \code{\link{joint_estimate}} for the method to jointly estimate the attachment function \eqn{A_k} and node fitnesses \eqn{\eta_i}.

}

\examples{
\dontrun{
  library("PAFit")
  set.seed(1)
  # size of initial network = 100
  # number of new nodes at each time-step = 100
  # Ak = k; inverse variance of the distribution of node fitnesse = 10
  net        <- generate_BB(N        = 1000 , m             = 50 ,
                            num_seed = 100  , multiple_node = 100,
                            s        = 10)

  net_stats  <- get_statistics(net)

  # estimate node fitnesses in isolation, assuming Ak = k
  result     <- only_F_estimate(net, net_stats)

  # plot the estimated node fitnesses and true node fitnesses
  plot(result, net_stats, true = net$fitness, plot = "true_f")
  }
}

\concept{fitness model}


#' @title FB2C
#' @description catch given Fishing mortality and Biomass; useful for  projections
#' @family abysmally documented
#' @author  unknown, \email{<unknown>@@dfo-mpo.gc.ca}
#' @export
#

FB2C <- function(F,B) {
	C = B*(exp(F)-1)*exp(-F)
	return(C)
}


#' cond.pr
#'
#' Estimate conditional probability for each observation
#' Default to use local method over knn method,
#' if both arguement supply or unspecify, only the local method will use.
#'
#' @param x vector or matrix whose rows contains predictors
#'          of each observations.
#' @param y vector of the variable we want to estimate its condtional cdf
#' @param span  fraction of total distance to be the bandwidth, between 0 and 1,
#'             default to be 0.5 if bandwidth not supply.
#' @param bandwidth the bandwidth of x to evaluate the cdf,
#'                 overwrite the span input.
#' @param rnn   fraction of total number of observations to be knn,
#'              between 0 and 1. Set span to be NULL to use knn,
#'              default to be 0.5, if not supply
#' @param knn   the number of nearest neighbor to estimate the cdf,
#'              should be less than the number of observation and greater than 0
#'              (\code{knn < length(y)}.
#'              setting this value overwrite the rnn
#'
#' @return vector of conditional probability \code{p(Y < y | x)}
#'
#' @examples
#' cond_pr(1:5, 1:5) #[1] 0.33 0.5 0.6 0.75 1
#' cond_pr(1:5, 5:1, span=0) #[1] 1 1 1 1 1
#' cond_pr(1:5, 5:1, span=1) #[1] 1.0 0.8 0.6 0.4 0.2
#' cond_pr(1:5, 5:1, span=NULL) [1] 1. 0.66 0.66 0.66 0.33
#' cond_pr(1:5, 5:1, span=NULL, rnn=1) [1] 1.0 0.8 0.6 0.4 0.2
#' cond_pr(1:5, 5:1, span=NULL, rnn=0) [1] 1 1 1 1 1
#' cond_pr(1:5, 5:1, span=NULL, rnn=0, knn=1) [1] 1. 0.6 0.66 0.66 0.5
#' cond_pr(1:5, 5:1, span=NULL, rnn=0, knn=3) [1] 1.00 0.75 0.60 0.50 0.25
#' cond_pr(1:5, 5:1, bandwidth=1) [1] 1 0.6 0.6 0.6 0.5
#' cond_pr(matrix(1:12, nrow=6), 6:1) [1] 1. 0.75 0.6 0.6 0.5 0.33
#' cond_pr(matrix(1:12, nrow=6), 6:1, span = 0.6) [1] 1.0.8 0.67 0.5 0.4 0.25
#' cond_pr(matrix(1:12, nrow=6), 6:1, bandwidth = 5) 1. 0.8 0.6 0.5 0.4 0.25
#' cond_pr(matrix(1:12, nrow=6), 6:1, span = NULL) [1] 1. 0.75 0.6 0.6 0.5 0.25
#' cond_pr(matrix(1:12, nrow=6), 6:1, span = NULL, rnn = 0)
#' cond_pr(matrix(1:12, nrow=6), 6:1, span = NULL, rnn = 1)
#' cond_pr(matrix(1:12, nrow=6), 6:1, span = NULL, rnn = 1, knn=1)

cond_pr <- function(x, y,
                    span = 0.1, bandwidth=NULL,
                    rnn = NULL, knn=NULL) {
    #--------------------------------------
    ### Checking---------------------------
    #--------------------------------------
    if (is.vector(x)) {
        if ((s_n <- length(x)) != length(y)) {
            stop("input error: length(x)!=length(y)!")
        }
        is_vec <- T
    } else {
        if ((s_n <- nrow(x)) != length(y)) {
            stop("input error: nrow(x)!=length(y)!")
        }
        is_vec <- F
    }

    use_local <- F
    if (!is.null(span) || !is.null(bandwidth)) {
        if ((span < 0 || span > 1) & missing(bandwidth)) {
            stop("input error: span out of bound, no bandwidth supply!")
        }
        use_local <- T
    } else {
        if (missing(knn) || is.null(knn)) {
            if (is.null(rnn)) {
                rnn <- 0.5
            } else if (rnn < 0 || rnn > 1) {
                stop("input error: rnn out of bound, no knn supply!")
            }
            knn <- floor(rnn * s_n)
        }

        if (knn >= s_n) {
            print("warning: knn > s_n, set knn = s_n.")
            return(order(y) / s_n)
        }
    }

    #---------------------------------------------------
    ### Calculate probability---------------------------
    #---------------------------------------------------
    if (use_local) {
        output <- local_method(s_n, x, y, span, bandwidth, is_vec)
    } else {
        output <- knn_method(s_n, x, y, knn, is_vec)
    }

    return(output)
}

local_method <- function(s_n, x, y, span, bandwidth, is_vec) {
    # preallocate vector for condition probability
    c_prob <- c()
    if (is_vec) {
        s_range <- max(x) - min(x)
        if (missing(bandwidth) || is.null(bandwidth)) {
            bandwidth <- span * s_range
        }
        if (bandwidth >= s_range) {
            print("warning: bandwidth > maxwidth, bandwidth = maxwidth.")
            return(order(y) / s_n)
        }
        # Estimate conditional probability
        for (i in seq_len(s_n)) {
            idx <- which(abs(x - x[i]) <= bandwidth)
            c_prob[i] <- mean(y[idx] <= y[i])
        }
    } else {
        # Preallocate matrix for distance
        m_dist <- matrix(0, nrow = s_n, ncol = s_n)
        # Estimate distance between each x_i
        for (i in seq_len(s_n)) {
            diff <- x[- (1:i), ] - tcrossprod(rep(1, s_n - i), x[i, ])
            m_dist[i, - (1:i)] <- rowSums(diff^2)
        }
        m_dist <- sqrt(m_dist)
        s_range <- max(m_dist) - min(m_dist)
        if (missing(bandwidth) || is.null(bandwidth)) {
            bandwidth <- span * s_range
        }
        if (bandwidth >= s_range) {
            print("warning: bandwidth > maxwidth, bandwidth = maxwidth.")
            return(order(y) / s_n)
        }
        m_dist <- m_dist + t(m_dist)
        for (i in seq_len(s_n)) {
            idx <- which(m_dist[i, ] <= bandwidth)
            c_prob[i] <- mean(y[idx] <= y[i])
        }
    }

    return(c_prob)
}

knn_method <- function(s_n, x, y, knn, is_vec) {
    # preallocate vector for condition probability
    c_prob <- c()
    if (is_vec) {
        ord <- order(x)
        # Estimate conditional probability
        for (i in seq_len(s_n)) {
            ord_i <- which(ord == i)
            l_idx <- max(1, ord_i - knn)
            u_idx <- min(s_n, ord_i + knn)
            set <- ord[l_idx:u_idx]
            v_dist <- abs(x[set] - x[i])
            # find the closest k elements
            kth <- sort(v_dist, partial = knn + 1)[knn + 1]
            idx <- which(v_dist <= kth)
            c_prob[i] <- mean(y[set[idx]] <= y[i])
        }
    } else {
        # Estimate distance between each x_i
        for (i in seq_len(s_n)) {
            diff <- abs(x - tcrossprod(rep(1, s_n), x[i, ]))
            v_dist <- rowSums(diff^2)

            kth <- sort(v_dist, partial = knn + 1)[knn + 1]
            idx <- which(v_dist <= kth)
            c_prob[i] <- mean(y[idx] <= y[i])
        }
    }

    return(c_prob)
}


##### Chapter 8: Association Rules -------------------

## Example: Identifying Frequently-Purchased Groceries ----
## Step 2: Exploring and preparing the data ----

# load the grocery data into a sparse matrix
library(arules)
groceries <- read.transactions("groceries.csv", sep = ",")
summary(groceries)

# look at the first five transactions
inspect(groceries[1:5])

# examine the frequency of items
itemFrequency(groceries[, 1:3])

# plot the frequency of items
itemFrequencyPlot(groceries, support = 0.1)
itemFrequencyPlot(groceries, topN = 20)

# a visualization of the sparse matrix for the first five transactions
image(groceries[1:5])

# visualization of a random sample of 100 transactions
image(sample(groceries, 100))

## Step 3: Training a model on the data ----
library(arules)

# default settings result in zero rules learned
apriori(groceries)

# set better support and confidence levels to learn more rules
groceryrules <- apriori(groceries, parameter = list(support =
                          0.006, confidence = 0.25, minlen = 2))
groceryrules

## Step 4: Evaluating model performance ----
# summary of grocery association rules
summary(groceryrules)

# look at the first three rules
inspect(groceryrules[1:3])

## Step 5: Improving model performance ----

# sorting grocery rules by lift
inspect(sort(groceryrules, by = "lift")[1:5])

# finding subsets of rules containing any berry items
berryrules <- subset(groceryrules, items %in% "berries")
inspect(berryrules)

# writing the rules to a CSV file
write(groceryrules, file = "groceryrules.csv",
      sep = ",", quote = TRUE, row.names = FALSE)

# converting the rule set to a data frame
groceryrules_df <- as(groceryrules, "data.frame")
str(groceryrules_df)


#' GEE Beta Equations
#'
#' Creates GEE Equations for beta coefficients.
#'
#' @param betas vector of coefficients
#' @param rho overdispersion parameter
#' @param dat matrix of multinomial clusters
#'
#' @return estimating equations for betas
create_beta_equations <- function (betas, rho, dat) {
  n <- dim(dat)[[2]]  # dimensions of multinom
  m <- n - 1  # order of multinom
  ni <- rowSums(dat) # cluster sizes
  num.clus <- dim(dat)[[1]]

  # helper vars
  betas.exp <- exp(betas)
  rho.sq <- rho ^ 2
  denom <- 1 + sum(betas.exp)
  dispi <- (ni * (1 + (ni - 1) * rho.sq))
  p.vec <- as.vector(betas.exp / denom)
  p.mat <- matrix(p.vec, num.clus, m, byrow = TRUE)

  # create matrices
  d.mat <- (diag( betas.exp * denom, m ) - betas.exp %*% t( betas.exp )) / (denom ^ 2)
  var.mat <- diag( p.vec, m ) - p.vec %*% t( p.vec )
  r.vec <- dat[, 1:m] - ni * p.mat

  # result
  d.mat %*% solve(var.mat) %*% rowSums( t((ni / dispi) * r.vec) )
}

#' GEE Rho Equations
#'
#' Creates GEE Equations for rho (overdispersion parameter).
#'
#' @param rho overdispersion parameter
#' @param betas vector of coefficients
#' @param dat matrix of multinomial clusters
#'
#' @return estimating equations for rho
create_rho_equations <- function (rho, betas, dat) {
  n <- dim(dat)[[2]]  # dimensions of multinom
  m <- n - 1  # order of multinom
  ni <- rowSums(dat) # cluster sizes
  num.clus <- dim(dat)[[1]]

  # helper vars
  betas.exp <- exp(betas)
  rho.sq <- rho ^ 2
  denom <- 1 + sum(betas.exp)
  dispi <- ni * (1 + (ni - 1) * rho.sq)
  p.vec <- as.vector(betas.exp / denom)
  p.mat <- matrix(p.vec, num.clus, m, byrow = TRUE)

  # create matrices
  var.mat <- diag(p.vec, m) - p.vec %*% t(p.vec)
  r.vec <- dat[, 1:m] - ni * p.mat
  res <- tr( r.vec %*% solve(var.mat) %*% t( (1 / dispi) * r.vec) ) - m * num.clus

  res
}


#!/usr/bin/Rscript

# M x N Chess Board.
M = 8; N = 8; board = matrix(0, nrow = M, ncol = N)

# Get/Set value on a board position.
getboard = function (position)    { board[position[1], position[2]] }
setboard = function (position, x) { board[position[1], position[2]] <<- x }

# (Relative) Hops of a Knight.
hops = cbind(c(-2, -1), c(-1, -2), c(+1, -2), c(+2, -1),
             c(+2, +1), c(+1, +2), c(-1, +2), c(-2, +1))

# Validate a move.
valid = function (move) {
    all(1 <= move & move <= c(M, N)) && (getboard(move) == 0)
}

# Moves possible from a given position.
explore = function (position) {
    moves = position + hops
    cbind(moves[, apply(moves, 2, valid)])
}

# Possible moves sorted according to their Wornsdorff cost.
candidates = function (position) {
    moves = explore(position)

    # No candidate moves available.
    if (ncol(moves) == 0) { return(moves) }

    wcosts = apply(moves, 2, function (position) { ncol(explore(position)) })
    cbind(moves[, order(wcosts)])
}

# Recursive function for touring the chess board.
knightTour = function (position, moveN) {

    # Tour Complete.
    if (moveN > (M * N)) {
        print(board)
        quit()
    }

    # Available moves.
    moves = candidates(position)

    # None possible. Backtrack.
    if (ncol(moves) == 0) { return() }

    # Make a move, and continue the tour.
    apply(moves, 2, function (position) {
                        setboard(position, moveN)
                        knightTour(position, moveN + 1)
                        setboard(position, 0)
                    })
}

# User Input: Starting position (in algebraic notation).
square = commandArgs(trailingOnly = TRUE)

# Convert into board co-ordinates.
row      = M + 1 - as.integer(substr(square, 2, 2))
ascii    = function (ch) { as.integer(charToRaw(ch)) }
col      = 1 + ascii(substr(square, 1, 1)) - ascii('a')
position = c(row, col)

# Begin tour.
setboard(position, 1); knightTour(position, 2)