## Things outside of the module won't get run when using the package.

#You can create vectors of random normals.
x = randn(40);
x'
# Adding the semicolon to the end hides the output.
# Vectors are column vectors by default, which take up a good chunk of screen.
# So often it's nicer to tranpose them.

#You index into arrays with brackets, like in R
x[3]
x[end]
x[2:5]
x[end-4:end-1]
x[36:end-1]

#You also index into strings that way.
greeting = "Hello world!"
greeting[3:4]
greeting[7:end-1]

#Or arrays of random uniforms.
u = rand(3,2,2)
u[:,:,1]

#Standard syntax for randomly sampling is to call rand on the thing you want to sample from.
rand(1:4, 10)

using Distributions
gamma_3_3 = Gamma(3, 3)
rand(gamma_3_3, 2, 4)

#Julia Arrays are like the arrays/vectors/matrices of R. They will also automatically promote
[3f0 4.0 2 π]

#The individual elements were
3f0 |> typeof
4.0 |> typeof
2 |> typeof
π |> typeof
#Note that pi is an irrational number! It automatically converts to whatever you want it to be.
2π
2π |> typeof
big(2)π
big(2)π |> typeof
# |> is the pipe operator in Julia.
# However, it isn't as powerful or fancy as magrittr's "%>%".
# You can look at Lazy.jl if you want more piping options.

#In "R" a scalar is really a length 1 vector.
#In julia, they are clearly distinct.
scalar4double = 4.0
zerodim4double = fill(4.0)
vector4double = fill(4.0, 1)
matrix4double = fill(4.0, 1, 1)
dim3array4double = fill(4.0, 1, 1, 1)

#They all print differently, but there's also a fundamental difference under the hood.
isbits(scalar4double)
isbits(zerodim4double)
isbits(vector4double)

#Q: Are only numbers bits?
isbits(gamma_3_3)

#"isbits" is low level, referring to how the memory is handled under the hood.
#"isbits" types are raw, they're handled directly, and an array of them will be one solid chunk of memory.
# This makes them
#If something is not bits, that normally means it
#Why would we ever want a zero dim array?
function f!(a::Number)
    a = 2.1
end
function f!(a::AbstractArray)
    a[:] = 2.1
end
f!(scalar4double)
scalar4double
f!(zerodim4double)
zerodim4double
f!(dim3array4double)
dim3array4double

# In Julia, variables are not copied.
# The first f! replaced the input number with another number, but the original scalar4double remained bound to 4
# The second f! modified the contents of the arrays
# The arrays stayed the same, but the contents changed.
# Be weary!
zerodim2 = zerodim4double
zerodim2[] = 17.0
zerodim2
zerodim4double
# Julia does not copy unless you ask it to!
# "=" binds a new name to the old variable.
zerodim3 = copy(zerodim2)
zerodim3[] = 9.4
zerodim2
zerodim3

# It is convention in Julia that whenever a function has side effects -- whenever it modifies inputs
# to end that function name with a "bang"!
# The mutated arguments are also placed first.
copy!(zerodim2, zerodim3)
zerodim2
fill!(zerodim2, -1e9)
zerodim4double

# Also, adding commas makes a column vector.
[3f0, 4.0, 2, π]
# For a matrix
[ 1 2 3 ; 4 5 6]
reshape(1:6, 2, 3)
# Note the differences! Order, and
# In Julia, ranges are lazy -- even when reshaped.

# Also, the colon operator also has low priority.
# What is this in R?
# using RCall
3*1:5+7
10:3:22
10:3:22 |> collect

# R is a lazy language, which is great.
# Julia is eager, but many functions and operations are implemented to be lazy.
# Array slices can also be lazy
@views x[4:7]
# On Julia 0.7, transposing matrices is also lazy. Not yet on 0.6.

#If Julia can't promote them all to the same type, it'll give up and create a vector that can hold anything.
[ 42im 9//2 "fish" π ]
#This isn't a good idea. Don't that.

#You can also create undefined arrays. Be cautious, they may contain NaNs, which can infect code if you're not careful!
Array{Float64}(4, 12)

#Or fill
fill(1.2, 4, 7)
fill(gamma_3_3, 2, 4)

#Or if you just want zeros
zeros(4,7)
zeros(Int, 4, 7)

#You can also use comprehensions. Remember x earlier?
x'
#Lets use it to create a design matrix!
X = [xᵢ^j for xᵢ ∈ x, j ∈ 0:4]

#Lets create some fake data! First, lets get a true beta:
β_true = [2, 3, -5, -2, 1.2]

#And then add a little noise
y = X * β_true .+ 0.1randn(40)

β_hat = X \ y

# "\" is the left devision operator.
# It just means use the thing on the left to divide the thing on the right
2 \ 4
# If we tranpose both X and y, we can get dimensions to line up for a right division
y' / X'

# This is of course equivalent to what we're all used to:
inv(X' * X) * X' * y

# Realstically, we'd always prefer to just use LM and immediately get convenient summary statistics.
using DataFrames, GLM
df = DataFrame(x = x, x² = X[:,3], x³ = X[:,4], x⁴ = X[:,5], y = y)
OLS = glm(@formula(y ~ x + x² + x³ + x⁴), df, Normal(), IdentityLink())

# There are three ways to define functions.
# For an anonymous function, use [arguments] -> [function body]
# For example, if we want a random positive definite matrix
Σ = randn(20,5) |> x -> x' * x

# If we have more than one argument, we need to wrap them in a parenthesis.
anon = (a, b) -> sin(a) - exp(b)
anon(2, 3)

# We can overwrite anon with anything.
anon = 4

# If we create a proper named function, we cannot replace it with something other than a function.
f(a) = 2a
f(4)
f = 3
f(a) = 3a
f(4)
f(a::Int) = "Hello world!"
f(a, b) = 3a + 2b
f(4)
f(4.0)
f(4, -5)

# Julia uses multiple dispatch.
# Julia compiles a different specialized version of that function for each combination of types you give it.
# It then calls the type matching that combination.
# This also lets you define a different body for the types if you want to.
# That is, Julia compiles 4 separate functions for "f" when we do this:
f(4, -5)
f(4.0, -5)
f(4, -5.0)
f(4.0, -5.0)
# And chooses the correct version to use.
# If we want the compiled function to act differently, we can then give Julia a different function body to use for that type
# Just like we did for "Hellow world!" above
# For example:
f(X, β_hat) #Error, dimensions don't line up!
f(a::Matrix, b::Vector) = b' * a' * a * b
f(X, β_hat)
f(4, -5) #Still works!
# Similar to match.call() in R:
pair = (4,-5)
f(pair...)

# The long form just makes it easier to write multi-line functions.
#@inline function f(x, y)
function f(x, y)
    out = zero(promote_type(typeof(x), eltype(y)))
    for yᵢ ∈ y
        out += x * yᵢ
    end
    out
end

@code_llvm f(e, x)

#This is a tuple. A tuple's type is partly defined by its length.
@code_llvm f(e, (1.2, -9.5, 3.2, 37.2))
(1.2, -9.5, 3.2, 37.2) |> typeof

@code_llvm f(e, 2.3)
@code_llvm f(2, 3)

function g(f, a, b, c)
    f(a, b) + c
end

# f is automatically inlined at -O3, not at -O2.
g(f, 2, 3, 4)
@code_llvm g(f, 2, 3, 4)

# In Julia, you end a scope or expression via "end".
f(√2) #Does not have the irrational type! It's just taking the square root of 2.

# R has "single dispatch", which is useful for things like plot, print, and summary methods.

# A simple example why this is useful in Julia is so that we can use "*" for matrix multiplication.
# There are a lot of different methods for "*"!
methods(*) |> length
#Run just methods(*) if you want to see the full list!

# This also makes code faster!
# Compare
S_big = randn(10_000, 5_000) |> x -> x' * x;
X_big = randn(5_000, 5_000);
@time S_big * X_big;
using Compat.LinearAlgebra #This line is currently unnecessary
@time eigfact(S_big);
# However, we know S is symmetric! What happens if we tell Julia that?
S_big2 = Symmetric(S);
@time S_big2 * X_big;
@time eigfact(S_big2);

#The using Compat.LinearAlgebra line is unnecassary in Julia 0.6
#However, starting in Julia 0.7 you'll need "using LinearAlgebra"
#Saying "using Compat.LinearAlgebra" works on both versions.

# Give Julia more information, and it will automatically pick the best method!

# Multiple dispatch is magical, because
# a) It makes code easier to read. When you want to multiply, you use "*".
# When you want an eigenfactorization, you use "eigfact".
# No "eigfact" here, "eigfact_symmetric" there.
# No map_int here and map_dbl there.

# However, what if we actually want elementise multiplication from "*"?
# In Julia, adding a "." automatically broadcosts.
# A broadcast is like an apply or map statement, except it will line up dimensions.

# Recall
size(X)
size(y)
y' * X # Regular matrix-vector product.
X .* y # Broadcast
# Both X and y have 40 rows, so it lines these up
# and then replicates y across each of X's 5 columns.
X .* β_hat'
# β_hat' is 1 x 5, so it lines up with X's 5 columns, and is replicated down the 40 rows.
X .+ 2
# Here, 2 gets replcicated along both colums and rows.

X .* y' # Does not line up -> error
X .* β_hat # Does not line up -> error

# This works for any function.
f.(X)
f.(X, y)

# If we use multiple ones, they will all fuse.
f.(X, y) .* β_hat' .+ 2

# Too many dots can be annoying to write. So Julia offers the macro "@. "
@. f(X, y) * β_hat + 2

# Whenever you see something with an "@" in Julia, it is a macro.
# Macros edit your code. Think of them as non-standard evaluation in R.
# Because they can do extremely weird things, they always have an "@" to let you know something funky is going on.
# A basic example of non-standard evaluation in R is how a lot of functions know the name of the variables you give them.
# Consider -- straight out of Advanced R:
"
x <- seq(0, 2 * pi, length = 100)
sinx <- sin(x)
plot(x, sinx, type = "l")
"
# Similarly:
@show anon;
# If you ever want to know what it is a macro does to your code, there's a macro for that!
@macroexpand @show anon
@macroexpand @. f(X, y) * β_hat + 2

# We can see that "@." does us the favor of adding "." to every function call for us.

# We can get creative!
bunch_o_gammas = @. Gamma(exp(f(X)), exp(-2y))

# What is the meaning of this? Good question!
mean.(bunch_o_gammas)
# Maybe we want to do parametric bootstrap?
samples = rand.(bunch_o_gammas, 4000)

# Method of Moment's estimator for Gamma
function gammas_MoM(x)
    x̄ = mean(x)
    s² = var(x)
    x̄^2 / s², s² / x̄
end
mom_estimators = gammas_MoM.(samples)
function relative_error( estimate, truth )
    @. ( estimate - truth ) / truth
end
#Relative errors
@. relative_error(mom_estimators, params(bunch_o_gammas))

#We can easily condense the steps
N = 4000
@. relative_error(gammas_MoM(rand(bunch_o_gammas, N)), params(bunch_o_gammas))

### Some more array manipulations and fun with broadcasting!
# Remember Σ from earlier?
Σ
# Let's take N samples from a multivariate normal with that as the covariance matrix, and mean:
μ = 2.* randn(5)

W = chol(Σ) * randn(5, N) .+ μ

mean(W, 2)
cov(W')
# We can also recall the definition from multivariate
S = (W .- mean(W,2)) |> x -> x * x' ./ (size(x,2)-1)
#and see that it is a pretty good estimator
Σ
# The matrix and broadcast syntax makes it easy to weave together array manipulations
# and apply definitions, like we see in Johnson and Wichern for the sample Covariance matrix.

# Remember this?
[ 42im 9//2 "fish" π ]
# What if we do want to hold a bunch of different types?
# One straightforward approach is to build our own container!

abstract type four_objects{A,B,C,D} end
struct FourThings{A,B,C,D} <: four_objects{A,B,C,D}
    a::A
    b::B
    c::C
    d::D
end
function FourThings(a::A, b::B, c::C, d::D) where {A,B,C,D}
    FourThings{A,B,C,D}(a,b,c,d)
end
ft1 = FourThings(42im, 9//2, "fish", π)
isbits(ft1)
ft2 = FourThings(42, 9//2,  4f0, π)
isbits(ft2)
mutable struct FourMutableThings{A,B,C,D} <: four_objects{A,B,C,D}
    a::A
    b::B
    c::C
    d::D
end
ft3 = FourMutableThings(42, 9//2,  4f0, π)
isbits(ft3)
struct MyMatrix{T}
    a::T
    b::T
    c::T
    d::T
    e::T
    f::T
    g::T
    h::T
    i::T
    j::T
    k::T
    l::T
    m::T
    n::T
    o::T
    p::T
end
# Then, we get define our operations. For example, multiplication
function Base.:*(x::MyMatrix{T}, y::four) where {T, A,B,C,D,four <: four_objects{A,B,C,D}}
    (
        x.a*y.a + x.e * y.b + x.i * y.c + x.m * y.d,
        x.b*y.a + x.f * y.b + x.j * y.c + x.n * y.d,
        x.c*y.a + x.g * y.b + x.k * y.c + x.o * y.d,
        x.d*y.a + x.h * y.b + x.l * y.c + x.p * y.d,
    )
end
v16 = randn(Float32, 16);
m16 = reshape(v16, 4, 4);
v4 = randn(Float32, 4);
mm = MyMatrix(v16...)
mm * ft2
mm * ft3
ft4 = FourThings(v4...)
ft5 = FourMutableThings(v4...)
m16 * v4
mm * ft4

# What do you expect their performance to be?
# Built in multiplication, vs our own custom version?
using BenchmarkTools
@benchmark $m16 * $v4
@benchmark $mm * $ft2
@benchmark $mm * $ft3
@benchmark $mm * $ft4
@benchmark $mm * $ft5
# Now compare with Julia launched with -O3
# This is why I like -O3

# More cool features
using ForwardDiff

ForwardDiff.derivative(f, 2.3)
ForwardDiff.derivative(sin, 2.3)
cos(2.3)


module KernelLibrary

export linear_kernel, polynomial_kernel, laplacian_kernel, gaussian_kernel, anova_kernel, rbf_kernel, fractrbf_kernel, rationalquad_kernel, multiquad_kernel, invmultiquad_kernel, circular_kernel, spherical_kernel, wave_kernel
export power_kernel, log_kernel, spline_kernel, cauchy_kernel, chisquare_kernel, histintersect_kernel, genhistintersect_kernel, gentstudent_kernel, fourier_kernel, sigmoid_kernel, wavelet_kernel, transinvwavelet_kernel
# Julia implementation of most of the kernel functions from
# crsouza.blogspot.com/2010/03/kernel-functions-for-machine-learning.html

function linear_kernel{T}(x::Array{T}, y::Vector{T}; c::Real=0.)
	c=convert(T,c)
	(y'*x .+ c)
end

function polynomial_kernel{T}(x::Array{T}, y::Vector{T}, d::Real; α::Real=1., c::Real=1.)
	d=convert(T,d); α=convert(T,α); c=convert(T,c);
	(α.*(y'*x) .+ c).^ d
end

function laplacian_kernel{T}(x::Array{T}, y::Vector{T}, σ::Real)
	σ=convert(T, σ)
	exp(-sum((x.-y).^2.,1).^.5 ./ σ)
end

function gaussian_kernel{T}(x::Array{T}, y::Vector{T}, σ::Real)
	σ=convert(T, σ)
	exp(-sum((x.-y).^2,1) ./ 2*σ.^ 2)
end

function anova_kernel{T}(x::Array{T}, y::Vector{T}, σ::Real, d::Real)
	d=convert(T,d); σ=convert(T, σ)
	sum(exp(-σ*(x.-y).^2).^d,1)
end

function rbf_kernel{T}(x::Array{T}, y::Vector{T}, σ::Real)
	σ=convert(T, σ)
	exp(-sum(abs(x.-y),1) ./ 2*σ.^ 2)
end

function fractrbf_kernel{T}(x::Array{T}, y::Vector{T}, σ::Real, fract::Real)
	σ=convert(T, σ); fract=convert(T, fract);
	exp(-sum(abs(x .- y).^fract,1).^(1/fract) ./ 2*σ.^ 2)
end

function rationalquad_kernel{T}(x::Array{T}, y::Vector{T}, c::Real)
	c = convert(T, c)
	n = sum((x.-y).^2,1)
	1 .- n./(n .+ c)
end

function multiquad_kernel{T}(x::Array{T}, y::Vector{T}, c::Real)
	c = convert(T, c)
	sqrt(sum((x.-y).^2,1) .+ c^2)
end

function invmultiquad_kernel{T}(x::Array{T}, y::Vector{T}, c::Real)
	c = convert(T, c)
	1 ./ sqrt(sum((x.-y).^2,1) .+ c^2)
end

function circular_kernel{T}(x::Array{T}, y::Vector{T}, σ::Real)
	σ=convert(T, σ)
	n = sqrt(sum((x.-y).^2,1))./σ
	out = (2/π).*(acos(-n) - n.*sqrt(1 .- n.^2))
	out[n.>=σ] = 0
	out
end

function spherical_kernel{T}(x::Array{T}, y::Vector{T}, σ::Real)
	σ=convert(T, σ)
	n = sqrt(sum((x.-y).^2,1))./σ
	out = 1 - 1.5.*n + .5.*n.^3
	out[n.>=σ] = 0
	out
end

function wave_kernel{T}(x::Array{T}, y::Vector{T}, θ::Real)
	θ=convert(T, θ)
	n = sqrt(sum((x.-y).^2,1))
	(θ./n).*sin(n./θ)
end

function power_kernel{T}(x::Array{T}, y::Vector{T}, d::Real)
	d=convert(T, d)
	-sqrt(sum((x.-y).^2,1)).^d
end

function log_kernel{T}(x::Array{T}, y::Vector{T}, d::Real)
	d=convert(T, d)
	-log(sqrt(sum((x.-y).^2.,1)).^d + 1)
end

function spline_kernel{T}(x::Array{T}, y::Vector{T})
	xy = x.*y
	mxy = broadcast(min, x,y)
    z = 1 .+ xy .+ xy.*mxy - ((x.+y)/2).*mxy.^2 + (mxy.^3)/3
    prod(z,1)
end

function cauchy_kernel{T}(x::Array{T}, y::Vector{T}, σ::Real)
	σ=convert(T, σ)
	1 ./ (1 .+ (sum((x.-y).^2,1) ./ σ.^2))
end

function chisquare_kernel{T}(x::Array{T}, y::Vector{T})
	1 .- sum((x.-y).^2 ./ ((x.+y)/2),1)
end

function histintersect_kernel{T}(x::Array{T}, y::Vector{T})
	sum(broadcast(min, x,y),1)
end

function genhistintersect_kernel{T}(x::Array{T}, y::Vector{T}, α::Real, β::Real)
	α=convert(T, α); β=convert(T, β);
	sum(broadcast(min, abs(x).^α,abs(y).^β),1)
end

function gentstudent_kernel{T}(x::Array{T}, y::Vector{T}, d::Real)
	d=convert(T, d)
	1 ./ (1.0 .+ sqrt(sum((x.-y).^2.,1)).^d)
end

function fourier_kernel{T}(x::Array{T}, y::Vector{T}, a::Real)
	a=convert(T, a)
    dist = x.-y;
    dist[dist.==0] = eps(eltype(x))
    z = sin((a + 1/2)*dist)./sin(dist/2);
    prod(z,1)
end

function sigmoid_kernel{T}(x::Array{T}, y::Vector{T}; α::Real=1, c::Real=0)
	α=convert(T, α); c=convert(T, c);
	tanh(α*y'*x + c)
end

motherwavelet(x::Array) = cos(1.75.*x).*exp(-(x.^2)./2)

function wavelet_kernel{T}(x::Array{T}, y::Vector{T}, a::Real, c::Real)
	a=convert(T, a); c=convert(T, c);
	prod(motherwavelet((x.-c)./a).*motherwavelet((y.-c)./a),1)
end

function transinvwavelet_kernel{T}(x::Array{T}, y::Vector{T}, a::Real)
	a=convert(T, a);
	prod(motherwavelet((x.-y)./a),1)
end

end # module


#### Similarity and dissimilarity measures between images ####

"""
`hausdorff_distance(imgA, imgB)` is the modified Hausdorff distance
between binary images (or point sets).

References
----------
Dubuisson, M-P; Jain, A. K., 1994. A Modified Hausdorff Distance for
Object-Matching.
"""
function hausdorff_distance(imgA::AbstractArray, imgB::AbstractArray)
  @assert size(imgA) == size(imgB) "Images must have the same dimensions"

  # trivial case
  isequal(imgA, imgB) && return 0.

  A, B = find(imgA), find(imgB)

  # return if there is no object to match
  (isempty(A) || isempty(B)) && return Inf

  # grid coordinates (ndims by npoints)
  A = hcat([Float64[ind2sub(size(imgA), a)...] for a in A]...)
  B = hcat([Float64[ind2sub(size(imgB), b)...] for b in B]...)

  m = size(A, 2); n = size(B, 2)

  D = zeros(m, n)
  for j=1:n, i=1:m
    @inbounds D[i,j] = norm(A[:,i] - B[:,j])
  end

  dAB = mean(minimum(D, 2))
  dBA = mean(minimum(D, 1))

  max(dAB, dBA)
end


using LinearAlgebra, Statistics, Plots

x_vals = [0 0 0; 2 -3 -4];
y_vals = [0 0 0; 4 3 -3.5];

plot(x_vals, y_vals, arrow = true, color = :blue,
     legend = :none, xlims = (-5, 5), ylims = (-5,5),
     annotations = [(2.2, 4.4, "[2, 4]"),
                    (-3.3, 3.3, "[-3, 3]"),
                    (-4.4, -3.85, "[-4, -3.5]")],
    xticks = -5:1:5, yticks = -5:1:5,
    framestyle = :origin)

 # Illustrate scalar multiplication
 x = [2];
 scalars = [-2 1 2];
 vals = [0 0 0; x * scalars]
 labels = [(-3.6, -4.2, "-2x"), (2.4, 1.8, "x"), (4.4, 3.8, "2x")]

 plot(vals, vals, arrow = true, color = [:red :red :blue],
     legend = :none, xlims = (-5, 5), ylims = (-5, 5),
     annotations = labels, xticks = -5:1:5, yticks = -5:1:5,
     framestyle = :origin)

x = ones(3);
y = [2, 4, 6];
x + y;
4x; # equivalent to 4 * x and 4 .* x

dot(x, y);
norm(x); # Norm of x
sqrt(sum(abs2, x)); # Give the same result

# fixed linear function, to generate a plane
f(x, y) = 0.2x + 0.1y;

# lines to vectors
x_vec = [0 0; 3 3]
y_vec = [0 0; 4 -4]
z_vec = [0 0; f(3, 4) f(3, -4)]

# draw the plane
n = 20
grid = range(-5, 5, length = n)
z2 = [ f(grid[row], grid[col]) for row in 1:n, col in 1:n ]
wireframe(grid, grid, z2, fill = :blues, gridalpha =1 )
plot!(x_vec, y_vec, z_vec, color = [:blue :green], linewidth = 3, labels = "",
      colorbar = false)

A = [1 2
     3 4]
A'
A = ones(3,3);
2I
A + I

# Solve Linear Equations
A = [1.0 2.0; 3.0 4.0];
y = ones(2,1);
det(A);
A_inv = inv(A)
x = A_inv * y # solution
A * x # should equal y (a vector of ones)
A \ y # produces the same solution

A = [1.0 2.0; 2.0 1.0];
evals, evecs = eigen(A);


"""
    uniformprobvec(n)

Return a uniform probability distribution vector of size `n`.
"""
function uniformprobvec(n)
    return ones(n) ./ n
end

"""
    uniformtransmat(n)

Return a stochastic matrix of size `n` with uniform transition probability distributions.
"""
function uniformtransmat(n)
    return ones(n, n) ./ n
end

"""
    randprobvec(n)

Return a random probability distribution vector of size `n`.
"""
function randprobvec(n)
    p = rand(n)
    return p ./ sum(p)
end

"""
    randtransmat(n)

Return a stochastic matrix of size `n` with random transition probability distributions.
"""
function randtransmat(n)
    P = rand(n, n)
    return P ./ sum(P, dims = 2)
end


# using LinearAlgebra
function cart2sph(x::T, y::T, z::T) where {T <: Real}
    hxy = hypot(x, y)
    r = hypot(hxy, z)
    pol = atan(hxy, z)
    az = atan(y, x)
    return [Float64(pol), Float64(az), Float64(r)]
end

function cart2sph(xyz::Point)
    x, y, z = xyz[1], xyz[2], xyz[3]
    return cart2sph(x, y, z)
end

function sph2cart(pol::T, az::T, r::T) where {T <: Real}
    x = r * sin(pol) * cos(az)
    y = r * sin(pol) * sin(az)
    z = r * cos(pol)
    return x, y, z
end
"""
Change array of vertex coordinates into array of vertex type
"""
function array_to_vertex(a::AbstractArray)
    return [Point3f0(vertex) for vertex in a]
end

"""
Change array of face into array of face type
"""
function array_to_faces(a::AbstractArray)
    return [TriangleFace{OffsetInteger{-1,UInt32}}(face) for face in a]
end

"""
Change array of array of into matrix
"""
function array_of_array_to_matrix(a::AbstractArray)
    return [a[i][j] for i = 1:length(a), j = 1:length(a[1])]
end
"""
Change matrix to array of array
"""
function matrix_to_array_of_array(mat::AbstractMatrix)
    return [mat[i, :] for i = 1:size(mat, 1)]
end # function]

"""
Change matrix to vertices
"""
function matrix_to_vertex(mat::AbstractMatrix)
    return [Point3f0(mat[i, :]) for i = 1:size(mat, 1)]
end # function

"""
Centering the coordinates of mesh vertices to their mean
"""
function centroid(v::AbstractMatrix)
    return sum(v, dims=1) / size(v, 1)
end
function centroid(verts::Vector{T1}) where {T1}
    return reduce(+, verts) / length(verts)
end

function mesh_centering(verts::Vector{T1}, faces::Vector{T2}) where {T1,T2}
    mean_coord = reduce(+, verts) / length(verts)
    new_verts = similar(verts)
    for i in eachindex(verts)
        new_verts[i] = verts[i] - mean_coord
    end
    return new_verts, faces
end
function mesh_centering(mesh::Mesh)
    verts = decompose(Point3f0, mesh)
    triangles = decompose(TriangleFace{Int}, mesh)
    verts, triangles = mesh_centering(verts, triangles)
    return Mesh(verts, faces(mesh))
end

function signed_volume_of_triangle(
    p1::T,
    p2::T,
    p3::T,
) where {T <: AbstractVector}
    return dot(p1, cross(p2, p3)) / 6.0
end

"""
These functions return the volume of mesh object
"""
function mesh_volume(
    verts::Array{Point{3,T1}},
    faces::Array{TriangleFace{T2}},
) where {T1 <: Real,T2 <: Int}
    total_vol = 0.0
    for face in faces
        vol = signed_volume_of_triangle(
            verts[face[1]],
            verts[face[2]],
            verts[face[3]],
        )
        total_vol += vol
    end
    return abs(total_vol)
end

function mesh_volume(mesh::Mesh)
    verts = decompose(Point3f0, mesh)
    faces = decompose(TriangleFace{Int}, mesh)

    return mesh_volume(verts, faces)
end
"""
These functions return the copy of mesh object with volume one
"""
function volume_normalizing(
    verts::Array{Point{3,T1}},
    faces::Array{TriangleFace{T2}},
) where {T1 <: Real,T2 <: Int}
    vol = mesh_volume(verts, faces)
    cbrt_vol = cbrt(vol)
    new_verts = similar(verts)
    for i in eachindex(verts)
        new_verts[i] = verts[i] / cbrt_vol
    end
    return new_verts, faces
end

function volume_normalizing2(
    verts::Array{Point{3,T1}},
    faces::Array{TriangleFace{T2}},
) where {T1 <: Real,T2 <: Int}
    vol = mesh_volume(verts, faces)
    cbrt_vol = cbrt(vol)
    new_verts = similar(verts)
    for i in eachindex(verts)
        new_verts[i] = verts[i] / cbrt_vol
    end
    return new_verts, faces
end

function volume_normalizing(mesh::Mesh)
    verts = decompose(Point3f0, mesh)
    triangles = decompose(TriangleFace{Int}, mesh)
    verts, triangles = volume_normalizing(verts, triangles)
    return Mesh(verts, faces(mesh))
end
"""
This function return the mesh object from verts and faces
"""
function create_mesh(verts, faces)
    faces = convert(Vector{TriangleFace{OffsetInteger{-1,UInt32}}}, faces)
    return Mesh(verts, faces)
end
"""
This function return the angle at vertex v1 from triangle(v1,v2,v3)
"""
function calculate_angle(v1, v2, v3)
    v12 = v2 - v1
    v13 = v3 - v1
    v12 = v12 / norm(v12)
    v13 = v13 / norm(v13)
    cos_val = dot(v12, v13)
    angle = acos(cos_val)
    return angle
end
# """
# This function return the internal angles of vertices
# """
# function internal_angles(verts,faces)
#     nrow,ncol = length(faces[1]),length(faces)
#     angles = Matrix{Float64}(undef,nrow,ncol)
#     for j in 1:ncol
#         f = faces[j]
#         for  i in 1:nrow
#             v1 = verts[f[i]]
#             idx2 = mod(i+1,nrow) == 0 ? 3 : mod(i+1,nrow)
#             idx3 = mod(i-1,nrow) == 0 ? 3 : mod(i-1,nrow)
#             # @show i
#             # @show idx3
#             # @show f
#             v2 = verts[f[idx2]]
#             v3 = verts[f[idx3]]
#
#             angles[i,j] = calculate_angle(v1,v2,v3)
#         end
#     end
#     return angles
# end
"""
This function return the internal angles of vertices
"""
function internal_angles_intrinsic(l)
    s23 = l[:, 1]
    s31 = l[:, 2]
    s12 = l[:, 3]

    a23 = acos.((s12.^2 + s31.^2 - s23.^2) ./ (2 * s12 .* s31))
    a31 = acos.((s23.^2 + s12.^2 - s31.^2) ./ (2 * s23 .* s12))
    a12 = acos.((s31.^2 + s23.^2 - s12.^2) ./ (2 * s31 .* s23))

    angles = [a23 a31 a12]
    return angles
end
function internal_angles(verts, faces)
    f1 = [face[1] for face in faces]
    f2 = [face[2] for face in faces]
    f3 = [face[3] for face in faces]

    s12 = norm.(verts[f2] - verts[f1])
    s13 = norm.(verts[f3] - verts[f1])
    s23 = norm.(verts[f3] - verts[f2])

    l = [s23 s13 s12]

    A = internal_angles_intrinsic(l)
end

"""
This function return the area of triangle made from vectors v1,v2, and v3)
"""
function calculate_area(v1, v2, v3)
    v12 = v2 - v1
    v13 = v3 - v1
    area = norm(cross(v12, v13)) / 2.0
    return area
end
"""
This function return the surface area)
"""
function surface_area(v::AbstractVector, f::AbstractVector)
    nv = length(v)
    nf = length(f)

    area = 0
    for i in 1:nf
        l1 = norm(v[f[i][2]] - v[f[i][3]])
        l2 = norm(v[f[i][3]] - v[f[i][1]])
        l3 = norm(v[f[i][1]] - v[f[i][2]])
        s = (l1 + l2 + l3) / 2.0
        square_area = s * (s - l1) * (s - l2) * (s - l3)
        if square_area < 0.0; square_area = 0.0 end

        area += sqrt(square_area)
    end
    return area
end

function surface_area_list(v::Matrix{<:AbstractFloat}, f::Matrix{<:Signed})
    f1 = @view f[:, 1]
    f2 = @view f[:, 2]
    f3 = @view f[:, 3]

    l1 = sqrt.(sum((view(v, f2, :) - view(v, f3, :)).^2, dims=2))
    l2 = sqrt.(sum((view(v, f3, :) - view(v, f1, :)).^2, dims=2))
    l3 = sqrt.(sum((view(v, f1, :) - view(v, f2, :)).^2, dims=2))
    s = (l1 + l2 + l3) / 2.0
    squared_area = s .* (s - l1) .* (s - l2) .* (s - l3)
    area = sqrt.(squared_area)
    return area
end
function surface_area(v::Matrix{<:AbstractFloat}, f::Matrix{<:Signed})
    area = surface_area_list(v, f)
    return sum(area)
end
function surface_area2(v::AbstractVector, f::AbstractVector)
    f1 = [face[1] for face in f]
    f2 = [face[2] for face in f]
    f3 = [face[3] for face in f]
    # F x 3 matrices of triangle edge vectors, named after opposite vertices
    v1 = [v[f3[i]] - v[f2[i]] for i in eachindex(f)]
    v2 = [v[f1[i]] - v[f3[i]] for i in eachindex(f)]
    # v3 = [v[f2[i]] - v[f1[i]] for i in eachindex(f)];
    n = cross.(v1, v2)
    area = norm.(n) / 2.0
    return sum(area)
end
function surface_area2(v::Matrix{<:AbstractFloat}, f::Matrix{<:Signed})
    f1 = @view f[:, 1]
    f2 = @view f[:, 2]
    f3 = @view f[:, 3]
    # F x 3 matrices of triangle edge vectors, named after opposite vertices
    v1 = view(v, f3, :) - view(v, f2, :)
    v2 = view(v, f1, :) - view(v, f3, :)
    # v3 = v[f2,:] - v[f1,:];
    n = [cross(v1[i, :], v2[i, :]) for i = 1:size(v1, 1)]
    area = norm.(n) / 2.0
    return sum(area)
end

function mesh_reorientation(mesh::Mesh, T, N, B)
    verts = decompose(Point3f0, mesh)
    mat_verts = array_of_array_to_matrix(verts)
    new_mat_verts = [T N B] \ mat_verts'
    new_mat_verts = new_mat_verts'
    new_verts = matrix_to_vertex(new_mat_verts)
    return Mesh(new_verts, faces(mesh))
end
function mesh_reorientation(mat_verts::AbstractMatrix, T, N, B)
    new_mat_verts = [T N B] \ mat_verts'
    return new_mat_verts'
end


using Random, Plots, StatsPlots, Distributions, DataFrames, Printf
# Define calculation of posterior
"""
Posterior Distribution (Gamma dist) for Poisson parameter λ

    Inputs
    ------
    data: data (0 or 1)
    α0: shape parameter of prior (Gamma dist)
    β0: inversed scale parameter of prior (Gamma dist)
    prob: credible interval probability (0 < prob < 1)

    Outputs
    -------
    results: posterior summary stats table (DataFrames)
    α_star: shape parameter of posterior (Gamma dist)
    β: inversed scale parameter (1 / θ) of posterior (Gamma dist)

"""
function poisson_stats(data, α0, β0, prob)
    # posterior
    n = size(data)[1]
    sum_data = sum(data)
    α_star = sum_data + α0
    β_star = n + β0
    θ_star = 1.0 / β_star
    d_posterior = Gamma(α_star, θ_star)
    # mean
    mean_λ = mean(d_posterior)
    # median
    median_λ = median(d_posterior)
    # mode
    mode_λ = mode(d_posterior)
    # Analytic Mode for Beta Dist
    # (a - 1.0) / (a + b - 2.0)
    # std
    sd_λ = std(d_posterior)
    # credible interval
    # ci upper boud
    ci_λ_upper = quantile(d_posterior, prob - (1-prob)/2)
    # ci lower boud
    ci_λ_lower = quantile(d_posterior, (1-prob)/2)
    ci_λ = [ci_λ_lower, ci_λ_upper]

    posterior_stats = reshape([mean_λ, median_λ, mode_λ, sd_λ, ci_λ], 1, 5)
    stats_string = ["Mean", "Median", "Mode", "SD", "CI"]
    results = DataFrame(posterior_stats, stats_string)
    return results, α_star, β_star
end

# Generate moc datas
λ = 3.0
n = 50
Random.seed!(99)
data = rand(Poisson(lam), n)

# Set prior
α0 = 1.0
β0 = 1.0

# Calculate posterior statistics
prob = 0.95
results, α_star, β_star = poisson_stats(data, α0, β0, prob)
print(results)

# Visualize Posterior
θ0, θ_star = 1.0/β0, 1.0/β_star
plot(Gamma(α0, θ0), xlims=(0, 6), ylims=(0, 1.75), xlabel="λ", ylabel="Probability Density", label=@sprintf("prior: Gamma(%.1f, %.1f)",α0, θ0), linestyle=:dash)
plot!(Gamma(α_star, θ_star), label=@sprintf("prior: Gamma(%.1f, %.2f)",α_star, θ_star), linestyle=:solid)


#Maximum Compatibility Pairing
using JuMP,Gurobi

#define the model
m =Model(Gurobi.Optimizer)

#Let n represents the number of cities to be paired
n=4

#declaring variables
@variable(m, x[1:n,1:n]>=0)

#define the compatibility coefficients
c =   [	80 65 83 77;
	54 87 61 66 ;
	92 45 53 59 ;
	70 61 81 76 ]

#define the objective function
@objective(m, Max, sum(c[i,j]* x[i,j] for i in 1:n for j in 1: n)) #objective is to maximize total compatibility pairing

#define constraints
for j = 1: n
  @constraint(m ,sum(x[i,j] for i in 1: n) ==1)
end

for i = 1: n
  @constraint(m ,sum(x[i,j] for j in 1: n) ==1 )
end

print(m)

#Printing the optimal solutions obtained
println("Optimal Solutions:")
status = JuMP.optimize!(m)
println(" Objective value: ",JuMP.objective_value(m))

println("Decision variable values:")
println("x = ", JuMP.value.(x))


### A Pluto.jl notebook ###
# v0.16.1

using Markdown
using InteractiveUtils

# ╔═╡ 35dd2eca-2446-11ec-1de6-0feeace9d4f7
md"""

#### Flash separation of a binary mixture of component A and component B

A liquid feed stream that contains 60% component A (1) and 40% component B (2) is separated in a Flash Drum operating at P = 1.21 MPa and T = 150$^{\circ}$C. The saturation pressures of pure component A and B can be modeled using the [Antoine equation](https://en.wikipedia.org/wiki/Antoine_equation):

$$\begin{eqnarray}
\ln\left(P_{1}^{sat}\right) &=& 15 -\frac{3010}{T+250}\\
\ln\left(P_{2}^{sat}\right) &=& 14 -\frac{2700}{T+205}
\end{eqnarray}$$

where $T$ denotes the temperature in units of celcius $^{\circ}$C and $P_{\star}^{sat}$ denotes the saturation pressure in units of kPa for component $\star$.

##### Assumptions

1. The liquid and vapor phases of all streams are ideal
1. The liquid and vapor phases inside the Flash Drum are ideal (Raoult's law)
1. The liquid and vapor phases inside the Flash Drum reach vapor-liquid equilibrium instantly

##### Solve:

 * Calculate the fraction of the input stream that exits the drum as liquid
 * Calculate the fraction of the input stream that exits the drum as vapor
 * Calculate the composition of the liquid ($x_{1}$ and $x_{2}$) and vapor ($y_{1}$ and $y_{2}$) in the exit streams

"""

# ╔═╡ 92ff1f65-d717-47a8-9cd3-af71c2855210
PlutoUI.LocalResource("./figs/Fig-Flash.png")

# ╔═╡ 98e43733-8711-4aa4-9081-697dd8865592
begin

	# write down some constants -
	T = 150 		# units: C
	P = 1210 		# units: kPa
	z₁ = 0.60 		# units: dimensionless
	z₂ = (1-z₁)		# units: dimensionless

	# return -
	nothing
end

# ╔═╡ 792e101b-e736-46d2-bbf3-a1197a0cc654
md"""

##### Governing equations

Let's start by writing down the governing equations. Starting from the total mol balance:

$$\dot{F} = \dot{L}+\dot{V}$$

we can divide both sides by $\dot{F}$ to give a relationship which constrains the fraction(s) of liquid and vapor in the outlet streams:

$$\hat{L}+\hat{V} = 1$$

Further, we know that the species mol balance around species $i$ is given by:

$$\dot{F}z_{i} = \dot{L}x_{i}+\dot{V}y_{i}\qquad{i=1,2}$$

However, because we assumed ideal vapor and liquid, we know that the liquid and vapor phase equlibrium compositions are related by the fugacity matching condition: $y_{i}P = x_{i}P_{i}^{sat}$. Lastly, we can sum the matching conditions to arrive at an expression which governs the pressure that is only in $x_{i}$:

$$P = x_{1}P_{1}^{sat} + x_{2}P_{2}^{sat}$$

which can be re-written (because we are in a binary mixture) as:

$$P = x_{1}P_{1}^{sat} + \left(1-x_{1}\right)P_{2}^{sat}$$

"""

# ╔═╡ 9e171e7c-953d-4a75-b6f6-cbae930dd1ab
md"""

#### Solve for $x_{1}$ and $x_{2}$:

We can re-arrange the pressure expression to solve for $x_{1}$:

$$x_{1} = \frac{P - P_{2}^{sat}}{P_{1}^{sat} -  P_{2}^{sat}}$$

"""

# ╔═╡ 8e13e386-6fbc-4bdf-8eed-de5bd35247da
# compute P1sat (units: kPa)
P1_sat = exp(15 - (3010)/(T+250))

# ╔═╡ 45a8e5fc-0929-496b-b01a-0e6f27f8b457
# compute P2sat (units: kPa)
P2_sat = exp(14 - (2700)/(T+205))

# ╔═╡ 2bd63a06-0eb6-4e4e-b2b0-78b0ac87e341
# solve for x₁:
x₁ = (P - P2_sat)/(P1_sat - P2_sat)

# ╔═╡ 6b3bcfa8-7487-4f84-abbe-185939d91a80
x₂ = 1 - x₁

# ╔═╡ 50a0b336-e14c-487a-9ba5-25a0024d9766
md"""
#### Solve for $y_{1}$ and $y_{2}$:

Once we have the liquid phase composition, we can use the fugacity matching condition to relate the liquid phase and the vapr phase becuase we have asssumed they are in equilibrium i.e.,

$$y_{i} = x_{i}\left(\frac{P_{i}^{sat}}{P}\right)\qquad{i=1,2}$$

"""

# ╔═╡ 7608d2c9-eb24-4db1-9fb3-452d52770e17
# solve for y₁:
y₁ = x₁ * (P1_sat/P)

# ╔═╡ 76f6cf32-8f87-4df0-b1db-a51ee102f709
# solve for y₂:
y₂ = x₂* (P2_sat/P)

# ╔═╡ ec5378ba-ca2c-4d31-b176-454b2cd99302
md"""

#### Solve for $\hat{L}$ and $\hat{V}$:

Now that we have the composition values in the exit streams, we can use the total mol and species mol balances to estimate the fraction
of the input feed that exited the unit as liquid $\hat{L}$ and vapur $\hat{V}$. If we divide both sides of the species mol balance by $\dot{F}$ we get the relationship:

$$z_{i} = \hat{L}x_{i}+\hat{V}y_{i}\qquad{i=1,2}$$

which we can use (along with the total mol balances) to solve for $\hat{L}$ and $\hat{V}$ i.e.,

$$\begin{eqnarray}
\hat{L}x_{1}+\hat{V}y_{1} &=& z_{1} \\
\hat{L}+\hat{V} &=& 1
\end{eqnarray}$$

This is a system of 2-equations and 2-unknowns ($\hat{L}$ and $\hat{V}$) that can be solved by substitution or by linear algebra.

###### Approach 1: Old school subsitution/elimination approach:

Let's rearrange the total mol expression to get $\hat{L}$ in terms of $\hat{V}$:

$$\hat{L} = 1 - \hat{V}$$

which can be subsituted into the species mol balance (let's use component 1):

$$z_{1} = \left(1 - \hat{V}\right)x_{1}+\hat{V}y_{1}$$

and solved for $\hat{V}$:

$$\hat{V} = \frac{z_{1} - x_{1}}{y_{1} - x_{1}}$$

Once we have $\hat{V}$, we can compute $\hat{L}$ from the total mol balance.
"""

# ╔═╡ bb56b509-5279-40f5-b898-e8c1b29dfb5e
# compute Vhat -
Vhat = (z₁ - x₁)/(y₁ - x₁)

# ╔═╡ 3f13e0a8-b58b-4ac7-899b-412154acf0cb
# what is Lhat -
Lhat = 1 - Vhat

# ╔═╡ 745177ba-a88d-496d-b0f6-2e9aca506866
md"""
###### Approach 2: Fancy linear algebra approach:

We can rewrite the 2$\times$2 system of equations into matrix-vector form and then compute the matrix inverse. In particlar,
the species and total mass balances for component 1 can be written as:

$$\begin{pmatrix}
x_{1} & y_{1} \\
1 & 1
\end{pmatrix}
\begin{pmatrix}
\hat{L} \\
\hat{V}
\end{pmatrix} =
\begin{pmatrix}
z_{1} \\
1
\end{pmatrix}$$

which is the form $\mathbf{A}\mathbf{x} = \mathbf{b}$, where the solution takes the form: $\mathbf{x} = \mathbf{A}^{-1}\mathbf{b}$.

"""

# ╔═╡ 4c0a79c3-87b4-4452-ba9e-0de5bd324ac5
# setup A -
A = [x₁ y₁ ; 1 1]

# ╔═╡ a835ed0a-a2d0-4a5b-a8a0-dffd7250e34b
# setup b -
bV = [z₁ ; 1]

# ╔═╡ 2bf7b85f-3118-4fa9-842d-f2c9f3d16dce
# compute the unknown (fractions of L and V)
x = inv(A)*bV

# ╔═╡ b091bbb6-13ce-4b1f-b36b-9964783767d4
html"""<style>
main {
    max-width: 1200px;
  	width: 80%;
	margin: auto;
	font-family: "Roboto, monospace";
}

a {
	color: blue;
	text-decoration: none;
}

.H1 {
    padding: 0px 30px;
}
</style>
"""

# ╔═╡ 64af17f4-c632-425c-a101-9dfa5e8cbd84
# julia setup -
using PlutoUI

# ╔═╡ 79f99ade-b514-4768-9bc9-bcfa281ce02f
png_joinpathsplit__FILE__1assetsimagepng = let
    import PlutoUI
    PlutoUI.LocalResource(joinpath(split(@__FILE__, '#')[1] * ".assets", "image.png"))
end

# ╔═╡ 00000000-0000-0000-0000-000000000001
PLUTO_PROJECT_TOML_CONTENTS = """
[deps]
PlutoUI = "7f904dfe-b85e-4ff6-b463-dae2292396a8"

[compat]
PlutoUI = "~0.7.14"
"""

# ╔═╡ 00000000-0000-0000-0000-000000000002
PLUTO_MANIFEST_TOML_CONTENTS = """
# This file is machine-generated - editing it directly is not advised

[[Base64]]
uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"

[[Dates]]
deps = ["Printf"]
uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"

[[HypertextLiteral]]
git-tree-sha1 = "72053798e1be56026b81d4e2682dbe58922e5ec9"
uuid = "ac1192a8-f4b3-4bfe-ba22-af5b92cd3ab2"
version = "0.9.0"

[[IOCapture]]
deps = ["Logging", "Random"]
git-tree-sha1 = "f7be53659ab06ddc986428d3a9dcc95f6fa6705a"
uuid = "b5f81e59-6552-4d32-b1f0-c071b021bf89"
version = "0.2.2"

[[InteractiveUtils]]
deps = ["Markdown"]
uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"

[[JSON]]
deps = ["Dates", "Mmap", "Parsers", "Unicode"]
git-tree-sha1 = "8076680b162ada2a031f707ac7b4953e30667a37"
uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
version = "0.21.2"

[[Logging]]
uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"

[[Markdown]]
deps = ["Base64"]
uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"

[[Mmap]]
uuid = "a63ad114-7e13-5084-954f-fe012c677804"

[[Parsers]]
deps = ["Dates"]
git-tree-sha1 = "a8709b968a1ea6abc2dc1967cb1db6ac9a00dfb6"
uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
version = "2.0.5"

[[PlutoUI]]
deps = ["Base64", "Dates", "HypertextLiteral", "IOCapture", "InteractiveUtils", "JSON", "Logging", "Markdown", "Random", "Reexport", "UUIDs"]
git-tree-sha1 = "d1fb76655a95bf6ea4348d7197b22e889a4375f4"
uuid = "7f904dfe-b85e-4ff6-b463-dae2292396a8"
version = "0.7.14"

[[Printf]]
deps = ["Unicode"]
uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"

[[Random]]
deps = ["Serialization"]
uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"

[[Reexport]]
git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b"
uuid = "189a3867-3050-52da-a836-e630ba90ab69"
version = "1.2.2"

[[SHA]]
uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"

[[Serialization]]
uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"

[[UUIDs]]
deps = ["Random", "SHA"]
uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"

[[Unicode]]
uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"
"""

# ╔═╡ Cell order:
# ╠═64af17f4-c632-425c-a101-9dfa5e8cbd84
# ╟─35dd2eca-2446-11ec-1de6-0feeace9d4f7
# ╟─92ff1f65-d717-47a8-9cd3-af71c2855210
# ╠═79f99ade-b514-4768-9bc9-bcfa281ce02f
# ╠═98e43733-8711-4aa4-9081-697dd8865592
# ╟─792e101b-e736-46d2-bbf3-a1197a0cc654
# ╟─9e171e7c-953d-4a75-b6f6-cbae930dd1ab
# ╠═8e13e386-6fbc-4bdf-8eed-de5bd35247da
# ╠═45a8e5fc-0929-496b-b01a-0e6f27f8b457
# ╠═2bd63a06-0eb6-4e4e-b2b0-78b0ac87e341
# ╠═6b3bcfa8-7487-4f84-abbe-185939d91a80
# ╟─50a0b336-e14c-487a-9ba5-25a0024d9766
# ╠═7608d2c9-eb24-4db1-9fb3-452d52770e17
# ╠═76f6cf32-8f87-4df0-b1db-a51ee102f709
# ╟─ec5378ba-ca2c-4d31-b176-454b2cd99302
# ╠═bb56b509-5279-40f5-b898-e8c1b29dfb5e
# ╠═3f13e0a8-b58b-4ac7-899b-412154acf0cb
# ╟─745177ba-a88d-496d-b0f6-2e9aca506866
# ╠═4c0a79c3-87b4-4452-ba9e-0de5bd324ac5
# ╠═a835ed0a-a2d0-4a5b-a8a0-dffd7250e34b
# ╠═2bf7b85f-3118-4fa9-842d-f2c9f3d16dce
# ╟─b091bbb6-13ce-4b1f-b36b-9964783767d4
# ╟─00000000-0000-0000-0000-000000000001
# ╟─00000000-0000-0000-0000-000000000002


using Distributions, CSV, DataFrames, LinearAlgebra, LaTeXStrings

# Load observed data
x_samples_df = CSV.read("gmm_observations.csv")
x_samples = hcat(x_samples_df[!, :X1], x_samples_df[!, :X2])

# Computes πᵢ for all i with respect to the given parameters in θ
# Inputs:
# - θ: A vector containing the current estimates of [α₀, μ₀, Σ₀, μ₁, Σ₁]
# - x: The (n x 2) matrix of observed X values
# Ouputs:
# 𝛑: An n dimensional vector where the ith coordinate is the value of πᵢ
function compute_all_πᵢ(θ, x)
    # Get dataset size
    n = size(x, 1)

    # Get current parameters
    α, μ₀, Σ₀, μ₁, Σ₁ = θ

    # Set distributions
    𝒩₁ = MvNormal(μ₁, Σ₁)
    𝒩₀ = MvNormal(μ₀, Σ₀)

    # Compute all πᵢ
    𝛑 = [α*pdf(𝒩₁, x[i,:])/(α*pdf(𝒩₁, x[i,:]) + (1-α)*pdf(𝒩₀, x[i,:])) for i=1:n]

    return 𝛑
end

# EM algorithm for GMM with observations x and Z ∈ {0,1}
# Inputs:
# - θ₀: The initial estimate of the θ parameter
# - x: The observed X values
# - iterations: The number of iterations for the EM algorithm
# Outputs:
# - θⱼ: The estimate of θ after 1000 iterations
function gmm_EM(θ₀, x, iterations)
    # Get dataset size
    n = size(x, 1)

    # Store the value of θ on jth step
    θⱼ = θ₀

    for j=1:iterations
        # Get current πᵢ values for all i
        𝛑 = compute_all_πᵢ(θⱼ, x)

        # Store the vector containing 1 - 𝛑
        𝛑₁ = 1 .- 𝛑

        # Get next value of α
        α = sum(𝛑)/n

        # Get next value of μ₀
        μ₀ = [dot(𝛑₁, x[:,1]), dot(𝛑₁, x[:,2])]/sum(𝛑₁)

        # Get next value of μ₁
        μ₁ = [dot(𝛑, x[:,1]), dot(𝛑, x[:,2])]/sum(𝛑)

        # Get next value of Σ₀.
        Σ₀ = sum([(x[i,:] - μ₀)*((x[i,:] - μ₀))'*𝛑₁[i] for i=1:n])/sum(𝛑₁)

        # Get next value of Σ₁.
        Σ₁ = sum([(x[i,:] - μ₁)*((x[i,:] - μ₁))'*𝛑[i] for i=1:n])/sum(𝛑)

        # Update θ
        θⱼ = [α, μ₀, Σ₀, μ₁, Σ₁]

    end

    return θⱼ
end

# Initialize EM parameters
α₀ = 0.3
μ₀ = [1.0, 2.0]
μ₁ = [2.0, 3.0]
Σ₀ = 1.0*Matrix(I, 2, 2)
Σ₁ = 1.0*Matrix(I, 2, 2)
θ₀ = [α₀, μ₀, Σ₀, μ₁, Σ₁]

println(gmm_EM(θ₀, x_samples, 20))

## Plots ##
gr(size=(300,300))

function ellipse!(μ,Σ;kw...)
    for i=1:3
        plot!([tuple((i*sqrt(Σ)*[cos(t),sin(t)]+μ)...) for t in range(0,stop=2π,length=100)];
              seriestype=:shape,linealpha=0.5,linecolor=:gray,fillalpha=[0.4,0.2,0.1][i],kw...)
    end
    current()
end

function mixtureplot(X₁,X₂,μ₀,Σ₀,μ₁,Σ₁,Π, plot_title)
    scatter(X₁,X₂;fillalpha=0.5,markerstrokewidth=0.5,
        marker_z=Π,mc=ColorGradient([:lightblue,:orange]),
        colorbar=:false, title=plot_title)
    ellipse!(μ₀,Σ₀,fillcolor=:lightblue)
    ellipse!(μ₁,Σ₁,fillcolor=:orange)
    plot!(;bg=:transparent,xlims=(-4.5,8.5),ylims=(-3.5,10.5),
        leg=false,ticks=:none,ratio=:equal)
end

one_step = gmm_EM(θ₀, x_samples, 1)
one_step_Π = compute_all_πᵢ(one_step, x_samples)
five_steps = gmm_EM(θ₀, x_samples, 5)
five_step_Π = compute_all_πᵢ(ten_steps, x_samples)
twenty_steps = gmm_EM(θ₀, x_samples, 20)
twenty_step_Π = compute_all_πᵢ(twenty_steps, x_samples)

# Zero step plot
mixtureplot(x_samples[:,1], x_samples[:,2], θ₀[2], θ₀[3], θ₀[4], θ₀[5], compute_all_πᵢ(θ₀, x_samples), L"\theta_0")
savefig("../images/gmm_mixtureplot_0.svg")

# One step plot
mixtureplot(x_samples[:,1], x_samples[:,2], one_step[2], one_step[3], one_step[4], one_step[5], one_step_Π,  L"\theta_1")
savefig("../images/gmm_mixtureplot_1.svg")

# Five step plot
mixtureplot(x_samples[:,1], x_samples[:,2], five_steps[2], five_steps[3], five_steps[4], five_steps[5], five_step_Π, L"\theta_5")
savefig("../images/gmm_mixtureplot_5.svg")

# Twenty step plot
mixtureplot(x_samples[:,1], x_samples[:,2], twenty_steps[2], twenty_steps[3], twenty_steps[4], twenty_steps[5], twenty_step_Π,  L"\theta_{20}")
savefig("../images/gmm_mixtureplot_20.svg")


module NumericalRange

using LinearAlgebra
using Plots

"""
        nrange(A; nk = 1, thmax = 32, noplot = false)

Numerical range (or field of values) of the matrix `A`.
Evaluates and plots the numerical range of the
`nk` largest leading principal submatrices of `A`, using `thmax`
equally spaced angles in the complex plane.
The defaults are `nk = 1` and `thmax = 32`
The eigenvalues of `A` are plotted as `x`.  The function returns `f` and `e`,
where `f` is the numerical range and `e` is a vector of eigenvalues of `A`.
Setting `noplot = true` suppresses the plot.

This function is a direct translation of `fv.m` in Professor Nick Higham's Matrix Computation Toolbox,
https://www.mathworks.com/matlabcentral/fileexchange/2360-the-matrix-computation-toolbox,
based on an original routine by A. Ruhe.

References:
R. A. Horn and C. R. Johnson, Topics in Matrix Analysis, Cambridge
     University Press, 1991; sec. 1.5.
A. S. Householder, The Theory of Matrices in Numerical Analysis,
     Blaisdell, New York, 1964; sec. 3.3.
C. R. Johnson, Numerical determination of the field of values of a
     general complex matrix, SIAM J. Numer. Anal., 15 (1978),
     pp. 595-602.

"""
function nrange(B; nk=1, thmax=32, noplot=false)
        function rq(A, x)
                return x'*A*x/(x'*x)
        end
        thmax -= 1 # the function uses thmax + 1 angles

        (n, p) = size(B)
        if n != p
                DimensionMismatch("Matrix must be square.")
        end

        z = Matrix{ComplexF64}(undef, 2*thmax + 1, 1)
        F = eigen(B)
        e = F.values
        f = []

        # filter out cases where B is Hermitian or skew-Hermitian
        if B == B'
                f = [minimum(e), maximum(e)]
        elseif B == -B'
                e = imag(e)
                f = [minimum(e), maximum(e)]
                e *= im
                f *= im
        else
                for m = 1:nk
                        ns = n + 1 - m
                        A = B[1:ns, 1:ns]
                        for i = 0:thmax
                                th = i/thmax*pi
                                Ath = exp(im*th)*A
                                H = 0.5*(Ath + Ath')
                                X = eigen(H, sortby = x -> real(x))
                                V = X.vectors
                                z[i + 1] = rq(A, V[:, 1])
                                z[i + 1 + thmax] = rq(A, V[:, ns])
                        end
                        f = [f; z]
                end
                # join up the boundary
                f = [f; f[1, :]]
        end
        if thmax == 0
                f = e
        end

        if !noplot
                p = plot(real(f), imag(f), framestyle=:origin, aspect_ratio = 1, legend=false)
                scatter!(p, real(e), imag(e), markershape=:x, legend=false)
                display(p)
        end

        return f, e
end

end


using LightGraphs, Distributions, StatsBase, Random, Plots, LaTeXStrings;pyplot()
Random.seed!(0)

function createNetwork(edges)
    network = Graph(maximum(maximum.(edges)))
    for e in edges
        add_edge!(network, e[1], e[2])
    end
    network
end

function uniformRandomEdge(network)
    outDegrees = length.(network.fadjlist)
    randI = sample(1:length(outDegrees),Weights(outDegrees))
    randJ = rand(network.fadjlist[randI])
    randI, randJ
end

function networkLife(network,source,dest,lambda)
    failureNetwork = copy(network)
    t = 0
    while has_path(failureNetwork, source, dest)
        t += rand(Exponential(1/(failureNetwork.ne*lambda)))
        i, j = uniformRandomEdge(failureNetwork)
        rem_edge!(failureNetwork, i, j)
    end
    t
end

lambda1, lambda2 = 0.5, 1.0
roads = [(1,2), (1,3), (2,4), (2,5), (2,3), (3,4), (3,5), (4,5), (4,6), (5,6)]
source, dest = 1, 6
network = createNetwork(roads)
N = 10^6

failTimes1 = [ networkLife(network,source,dest,lambda1) for _ in 1:N ]
failTimes2 = [ networkLife(network,source,dest,lambda2) for _ in 1:N ]

println("Edge Failure Rate = $(lambda1): Mean failure time = ",
	mean(failTimes1), " days.")
println("Edge Failure Rate = $(lambda2): Mean failure time = ",
	mean(failTimes2), " days.")

stephist(failTimes1, bins=200, c=:blue, normed=true, label=L"\lambda=0.5")
stephist!(failTimes2, bins=200, c=:red, normed=true, label=L"\lambda=1.0",
    xlims=(0,5), ylims=(0,1.1), xlabel="Network Life Time", ylabel = "Density")


### A Pluto.jl notebook ###
# v0.12.20

using Markdown
using InteractiveUtils

# ╔═╡ b9f84364-6b75-11eb-183f-9358dcc0e4d0
begin
	using Test
	using LinearAlgebra
end

# ╔═╡ 9c44adc6-6b7c-11eb-20b5-11ad83fee84b
ArrayFloatOrIntDim2 = Array{<:Union{Float64, Int64}, 2}

# ╔═╡ 95a22bc2-6b77-11eb-2cd5-c50658a942b9
md"""
## Gram - Schmidt
"""

# ╔═╡ 3d8a1668-6b70-11eb-293c-ab5ce07a3ce5
begin
	function GramSchmidt(A::ArrayFloatOrIntDim2)
		m, n = size(A)
		Q = zeros(m, n)
		R = zeros(n, n)
		for j = 1:n
			v = A[:,j]
			for i = 1:j-1
				R[i, j] = Q[:, i]' * v
				v = v - R[i, j] * Q[:, i]
			end
			R[j, j] = norm(v)
			Q[:,j] = v / R[j,j]
		end
		return (Q, R)
	end
end

# ╔═╡ 7356cb64-6b71-11eb-01f7-91f369a9fb74
begin
	A = [
		1 0
		1 1
		1 3
		1 4
	]
	Q, R = GramSchmidt(A)

	@test A == round.(Int, Q * R)
end

# ╔═╡ 78c7863c-6b7c-11eb-0be7-cbfb3c368065
md"""
## Householder (probably wrong)
"""

# ╔═╡ 83f357d4-6b7c-11eb-11b6-5ddf77041fe5
begin
	function householder(A::ArrayFloatOrIntDim2)
		A = convert(Array{Float64}, A)
		m, n = size(A)
		U = zeros(m, n)
		for k = 1:n
			w = A[k:m, k]
			wₙ = norm(w)
			w[1] = (w[1] - wₙ)
			u = w / wₙ
			U[k:m, k] = u
			A[k:m, k:n] = A[k:m, k:n] - 2 * u * (u' * A[k:m, k:n])
		end
		R = triu(A[:, 1:n])
		return U, R
	end
end

# ╔═╡ 43133fa8-6b7d-11eb-331e-73d6eabd7d83
begin
	U, Rₕ = householder(A)
end

# ╔═╡ Cell order:
# ╠═b9f84364-6b75-11eb-183f-9358dcc0e4d0
# ╠═9c44adc6-6b7c-11eb-20b5-11ad83fee84b
# ╟─95a22bc2-6b77-11eb-2cd5-c50658a942b9
# ╠═3d8a1668-6b70-11eb-293c-ab5ce07a3ce5
# ╠═7356cb64-6b71-11eb-01f7-91f369a9fb74
# ╟─78c7863c-6b7c-11eb-0be7-cbfb3c368065
# ╠═83f357d4-6b7c-11eb-11b6-5ddf77041fe5
# ╠═43133fa8-6b7d-11eb-331e-73d6eabd7d83


"""
    copy_eigenvalues!(λs, A) → λs

Puts the eigenvalues of a quasi-upper triangular matrix A in the λs vector.
"""
function copy_eigenvalues!(λs, A::AbstractMatrix{T}, range = OneTo(size(A, 2)), tol = eps(real(T))) where {T}
    i = first(range)

    @inbounds while i < last(range)
        if is_offdiagonal_small(A, i, tol)
            λs[i] = A[i, i]
            i += 1
        else
            # Conjugate pair
            d = A[i,i] * A[i+1,i+1] - A[i,i+1] * A[i+1,i]
            x = (A[i,i] + A[i+1,i+1]) / 2
            y = sqrt(complex(x*x - d))
            λs[i + 0] = x + y
            λs[i + 1] = x - y
            i += 2
        end
    end

    @inbounds if i == last(range)
        λs[i] = A[i, i]
    end

    return λs
end

"""
    eigenvalue(R, i) → λ

Get the `i`th eigenvalue of R. NOTE: assumes `i` points to the start of
a block.
"""
function eigenvalue(R, i)
    n = minimum(size(R))

    @inbounds begin
        if i == n || iszero(R[i+1,i])
            return complex(R[i,i])
        else
            d = R[i,i] * R[i+1,i+1] - R[i,i+1] * R[i+1,i]
            x = (R[i,i] + R[i+1,i+1]) / 2
            y = sqrt(complex(x*x - d))
            return x + y
        end
    end
end

"""
    eigenvalues(A::AbstractMatrix{T}) → Vector{complex(T)}

Computes the eigenvalues of the matrix A. Assumes that A is quasi-upper triangular.
The eigenvalues are returned in complex arithmetic, even if their imaginary
part is 0.
"""
eigenvalues(A::AbstractMatrix{T}, tol = eps(real(T))) where {T} =
    copy_eigenvalues!(Vector{complex(T)}(undef, size(A, 2)), A, OneTo(size(A, 2)), tol)

"""
    partialeigen(P::PartialSchur) → (Vector{<:Union{Real,Complex}}, Matrix{<:Union{Real,Complex}})

Transforms a partial Schur decomposition into an eigendecomposition.

!!! note

    For real-symmetric and Hermitian matrices the Schur vectors coincide with
    the eigenvectors, and hence it is not necessary to call this function in
    that case.

The method still relies on LAPACK to compute the eigenvectors of the (quasi)
upper triangular matrix `R` from the partial Schur decomposition.

!!! note

    This method is currently type unstable for real matrices, since we have not
    yet decided how to deal with complex conjugate pairs of eigenvalues. E.g.
    if almost all eigenvalues are real, but there are just a few conjugate
    pairs, should all eigenvectors be complex-valued?
"""
function partialeigen(P::PartialSchur)
    vals, vecs = eigen(P.R)
    return vals, P.Q*vecs
end


# Description:
# Dimensions: d
#
# The Michalewicz function has d! local minima, and it is multimodal. The parameter m defines the steepness of they valleys and ridges; a larger m leads to a more difficult search. The recommended value of m is m = 10. The function's two-dimensional form is shown in the plot above.
#
# Input Domain:
# The function is usually evaluated on the hypercube xi ∈ [0, π], for all i = 1, …, d.
#
# Global Minima:
# https://www.sfu.ca/~ssurjano/michal3.png

function michal(xx, m = 10)

    d = length(xx);
    sum = 0;

    for ii = 1:d
       	xi = xx[ii];
       	new = sin(xi) * (sin(ii * xi^2 / pi))^(2 * m);
       	sum  = sum + new;
    end

    y = -sum;

    return [y]
end


# Load Julia packages (libraries) needed

using StatisticalRethinking
#gr(size=(600,600));

# ### snippet 2.3

# Define a grid

N = 20
p_grid = range( 0 , stop=1 , length=N )

# Define the (uniform) prior

prior = ones( 20 );

# Compute likelihood at each value in grid

likelihood = [pdf(Binomial(9, p), 6) for p in p_grid]
likelihood[1:5]

# Compute product of likelihood and prior

unstd_posterior = likelihood .* prior;

# Standardize the posterior, so it sums to 1

posterior = unstd_posterior  ./ sum(unstd_posterior);

# ### snippet 2.4

p1 = plot( p_grid , posterior ,
    xlab="probability of water" , ylab="posterior probability",
    lab = "interpolated", title="20 points" )
p2 = scatter!( p1, p_grid , posterior, lab="computed" )

# ### snippet 2.5

prior1 = [p < 0.5 ? 0 : 1 for p in p_grid]
prior2 = [exp( -5*abs( p - 0.5 ) ) for p in p_grid]

p3 = plot(p_grid, prior1,
  xlab="probability of water" , ylab="posterior probability",
  lab = "semi_uniform", title="Other priors" )
scatter!(p3, p_grid, prior1, lab = "semi_uniform grid point")
plot!(p3, p_grid, prior2,  lab = "double_exponential" )
scatter!(p3, p_grid, prior2,  lab = "double_exponential grid point" )

# End of `02/clip-03-05.jl`


using LinearAlgebra
using Plots
using Random
using BenchmarkTools
using JuMP

function kernelf(X1, X2, l=1.0, sigma_f=1.0)

    # Isotropic squared exponential kernel. Computes
    # a covariance matrix from points in X1 and X2.

    #Σ = zeros(size(X1,1), size(X2,1))
    Σ = Array{Float64,2}(undef, size(X1,1), size(X2,1))
    for i in 1:size(Σ,1)
        for j in 1:size(Σ,2)
            Σ[i,j] = sigma_f^2*exp(-(X1[i]-X2[j])^2/(2*l^2))
        end
    end

    return Σ

end

function plot_gp(μ, Σ, X, X_train=nothing, Y_train=nothing)
    σ = 1.96*sqrt.(diag(Σ))
    p = plot([X X], [μ μ], fillrange=[μ+σ μ-σ], fillalpha=0.3, c=:blue, label="")
    plot!(X, μ, c=:blue, label="GP mean")
    if X_train!=nothing && Y_train!=nothing
        plot!(X_train, Y_train, seriestype="scatter", color="red", label="samples")
    end
    display(p)
end

function kernelTest()
    X1 = collect(range(-5, step=1, 5))
    X2 = collect(range(-5, step=1, 5)) .+ 0.1.*randn(size(X1))
    plot(X1, label="original data")
    plot!(X2, label="corrupted data")
    μ = zeros(size(X1))
    Σ = kernelf(X1, X1)
    plot_gp(μ, Σ, X1)
end

function posteriorPredictor(X_s, X_train, Y_train, l=1.0, sigma_f=1.0, sigma_y=1e-8, K_inv=nothing)
    K_s = kernelf(X_train, X_s, l, sigma_f)
    K_ss = kernelf(X_s, X_s, l, sigma_f) + 1e-8*I
    if K_inv == nothing
        K = kernelf(X_train, X_train, l, sigma_f) + sigma_y^2*I
        K_inv = inv(K)
    end

    μ_s = K_s'*K_inv*Y_train
    #mu_s = K_s.T.dot(K_inv).dot(Y_train)
    Σ_s = K_ss - K_s'*K_inv*K_s
    #cov_s = K_ss - K_s.T.dot(K_inv).dot(K_s)

    return μ_s, Σ_s, K_inv
end

function testPredictor()
    lb = 0;
    ub = 5;
    n = 5;
    sigma_y = 0.0
    X_train = [-4 -3 -2 -1 1 4]'
    X_train = rand(n) .*(ub-lb) .+lb
    Y_train = sin.(X_train) + sigma_y*randn(size(X_train,1))
    X = collect(range(lb, ub, length=n))
    μ_s, Σ_s = posteriorPredictor(X, X_train, Y_train, 1.0, 1.0, sigma_y)
    plot_gp(μ_s, Σ_s, X, X_train, Y_train)
    plot!(X, sin.(X), lw=:2, color=:black, label="true function")
end

function hyperParameterOptimization(X_train, Y_train, noise, naive=true)
    model = Model()
    @variable(theta[1:2] >= 1e-5) #hyperparam variables
    if naive
    end
end

function f(x)
    return 8*sin.(x) + 1/2*x.^2 + 1/8*x.^3 + 1/32*x.^4 - 1/128*x.^5
end

#kernelTest()
#testPredictor()

lb = 1;
ub = 7;
n = 100;
l = 1.0;
sigma_f = 1.0
sigma_y = 0.9
#X_train = [-4 -3 -2 -1 1 4]'
#X_train = rand(n).*(ub-lb) .+ lb
X_train = 1.1*randn(n) .+ (ub - lb)/2 .+ lb
Y_train = f(X_train) + sigma_y*randn(size(X_train,1))
X = collect(range(lb, ub, length=n))
if sigma_y != 0.0
    μ_s, Σ_s, K_inv = posteriorPredictor(X, X_train, Y_train, l, sigma_f, sigma_y)
else
    μ_s, Σ_s, K_inv = posteriorPredictor(X, X_train, Y_train, l, sigma_f)
end
println("Timing training + inference")
@time posteriorPredictor(X, X_train, Y_train, 1.0, 1.0, sigma_y)
println("Timing inference")
@time posteriorPredictor(X, X_train, Y_train, 1.0, 1.0, sigma_y, K_inv)
plot_gp(μ_s, Σ_s, X, X_train, Y_train)
plot!(X, f(X), lw=:2, color=:black, label="true function", xlim=(lb,ub))


# Integration using a function
function trapz(f, a, b, h)
    x = collect(Float64, a:h:b);
    suma = 0
    for j ∈ 2:(length(x) - 1)
        val = f(x[j])
        suma += val
    end
    I = (b - a) * (f(x[1]) + 2 * suma + f(x[end])) / (2 * (length(x) - 1));
    return I
end

# Integration of a table
function trapz(x::Array{Float64,1}, y::Array{Float64,1})
    suma = 0;
    for j ∈ 2:(length(x) - 1)
        suma += y[j]
    end
    I = (x[end] - x[1]) * (y[1] + 2 * suma + y[end]) / (2 * (length(x) - 1));
    return I
end


# some simpler (and faster) implementations for root finding
#
# Not exported
#
# These avoid the setup costs of the `find_zero` method, so should be faster
# though they will take similar number of function calls.
#
# `Roots.bisection(f, a, b)`  (Bisection).
# `Roots.secant_method(f, xs)` (Order1) secant method
# `Roots.dfree(f, xs)`  (Order0) more robust secant method
#

## Bisection
##
## Essentially from Jason Merrill https://gist.github.com/jwmerrill/9012954
## cf. http://squishythinking.com/2014/02/22/bisecting-floats/
## This also borrows a trick from https://discourse.julialang.org/t/simple-and-fast-bisection/14886
## where we keep x1 so that y1 is negative, and x2 so that y2 is positive
## this allows the use of signbit over y1*y2 < 0 which avoid < and a multiplication
## this has a small, but noticeable impact on performance.
"""
    bisection(f, a, b; [xatol, xrtol])

Performs bisection method to find a zero of a continuous
function.

It is assumed that (a,b) is a bracket, that is, the function has
different signs at a and b. The interval (a,b) is converted to floating point
and shrunk when a or b is infinite. The function f may be infinite for
the typical case. If f is not continuous, the algorithm may find
jumping points over the x axis, not just zeros.

If non-trivial tolerances are specified, the process will terminate
when the bracket (a,b) satisfies `isapprox(a, b, atol=xatol,
rtol=xrtol)`. For zero tolerances, the default, for Float64, Float32,
or Float16 values, the process will terminate at a value `x` with
`f(x)=0` or `f(x)*f(prevfloat(x)) < 0 ` or `f(x) * f(nextfloat(x)) <
0`. For other number types, the A42 method is used.

"""
function bisection(f, a::Number, b::Number; xatol=nothing, xrtol=nothing)

    x1, x2 = adjust_bracket(float.((a,b)))
    T = eltype(x1)

    atol = xatol == nothing ? zero(T) : abs(xatol)
    rtol = xrtol == nothing ? zero(one(T)) : abs(xrtol)
    CT = iszero(atol) && iszero(rtol) ?  Val(:exact) : Val(:inexact)

    x1, x2 = float(x1), float(x2)
    y1, y2 = f(x1), f(x2)

    _unitless(y1 * y2) >= 0  && error("the interval provided does not bracket a root")

    if isneg(y2)
        x1, x2, y1, y2 = x2, x1, y2, y1
    end

    xm = Roots._middle(x1, x2) # for possibly mixed sign x1, x2
    ym = f(xm)

    while true

        if has_converged(CT, x1, x2, xm, ym, atol, rtol)
            return xm
        end

        if isneg(ym)
            x1, y1 = xm, ym
        else
            x2, y2 = xm, ym
        end

        xm = Roots.__middle(x1,x2)
        ym = f(xm)

    end

end

# -0.0 not returned by __middle, so isneg true on [-Inf, 0.0)
@inline isneg(x::T) where {T <: AbstractFloat} = signbit(x)
@inline isneg(x) = _unitless(x) < 0

@inline function has_converged(::Val{:exact}, x1, x2, m, ym, atol, rtol)
    iszero(ym) && return true
    isnan(ym) && return true
    x1 != m && m != x2 && return false
    return true
end

@inline function has_converged(::Val{:inexact}, x1, x2, m, ym, atol, rtol)
    iszero(ym) && return true
    isnan(ym) && return true
    val = abs(x1 - x2) <= atol + max(abs(x1), abs(x2)) * rtol
    return val
end

"""
    secant_method(f, xs; [atol=0.0, rtol=8eps(), maxevals=1000])

Perform secant method to solve f(x) = 0.

The secant method is an iterative method with update step
given by b - fb/m where m is the slope of the secant line between
(a,fa) and (b,fb).

The inital values can be specified as a pair of 2, as in `(a,b)` or
`[a,b]`, or as a single value, in which case a value of `b` is chosen.

The algorithm returns m when `abs(fm) <= max(atol, abs(m) * rtol)`.
If this doesn't occur before `maxevals` steps or the algorithm
encounters an issue, a value of `NaN` is returned. If too many steps
are taken, the current value is checked to see if there is a sign
change for neighboring floating point values.

The `Order1` method for `find_zero` also implements the secant
method. This one will be faster, as there are fewer setup costs.

Examples:

```julia
Roots.secant_method(sin, (3,4))
Roots.secant_method(x -> x^5 -x - 1, 1.1)
```

Note:

This function will specialize on the function `f`, so that the inital
call can take more time than a call to the `Order1()` method, though
subsequent calls will be much faster.  Using `FunctionWrappers.jl` can
ensure that the initial call is also equally as fast as subsequent
ones.

"""
function secant_method(f, xs; atol=zero(float(real(first(xs)))), rtol=8eps(one(float(real(first(xs))))), maxevals=100)

    if length(xs) == 1 # secant needs x0, x1; only x0 given
        a = float(xs[1])

        h = eps(one(real(a)))^(1/3)
        da = h*oneunit(a) + abs(a)*h^2 # adjust for if eps(a) > h
        b = a + da

    else
        a, b = promote(float(xs[1]), float(xs[2]))
    end
    secant(f, a, b, atol, rtol, maxevals)
end

function secant(f, a::T, b::T, atol=zero(T), rtol=8eps(T), maxevals=100) where {T}
    nan = (0a)/(0a)
    cnt = 0

    fa, fb = f(a), f(b)
    fb == fa && return nan

    uatol = atol / oneunit(atol) * oneunit(real(a))
    adjustunit = oneunit(real(fb))/oneunit(real(b))

    while cnt < maxevals
        m = b - (b-a)*fb/(fb - fa)
        fm = f(m)

        iszero(fm) && return m
        isnan(fm) && return nan
        abs(fm) <= adjustunit * max(uatol, abs(m) * rtol) && return m
        if fm == fb
            sign(fm) * sign(f(nextfloat(m))) <= 0 && return m
            sign(fm) * sign(f(prevfloat(m))) <= 0 && return m
            return nan
        end

        a,b,fa,fb = b,m,fb,fm

        cnt += 1
    end

    return nan
end

"""
    muller(f, xᵢ; xatol=nothing, xrtol=nothing, maxevals=100)
    muller(f, xᵢ₋₂, xᵢ₋₁, xᵢ; xatol=nothing, xrtol=nothing, maxevals=100)

> *Muller’s method* generalizes the secant method, but uses quadratic
> interpolation among three points instead of linear interpolation between two.
> Solving for the zeros of the quadratic allows the method to find complex
> pairs of roots.
> Given three previous guesses for the root `xᵢ₋₂`, `xᵢ₋₁`, `xᵢ`, and the values
> of the polynomial `f` at those points, the next approximation `xᵢ₊₁` is produced.

Excerpt and the algorithm taken from

> W.H. Press, S.A. Teukolsky, W.T. Vetterling and B.P. Flannery
> *Numerical Recipes in C*, Cambridge University Press (2002), p. 371

Convergence here is decided by `xᵢ₊₁ ≈ xᵢ` using the tolerances specified,
which both default to `eps(one(typeof(abs(xᵢ))))^4/5` in the appropriate units.
Each iteration performs three evaluations of `f`.
The first method picks two remaining points at random in relative proximity of `xᵢ`.

Note that the method may return complex result even for real intial values
as this depends on the function.

Examples:
```
muller(x->x^3-1, 0.5, 0.5im, -0.5) # → -0.500 + 0.866…im
muller(x->x^2+2, 0.0, 0.5, 1.0) # → ≈ 0.00 - 1.41…im
muller(x->(x-5)*x*(x+5), rand(3)...) # → ≈ 0.00
muller(x->x^3-1, 1.5, 1.0, 2.0) # → 2.0, Not converged
```

"""
muller(f, x₀::T; kwargs...) where T = muller(f, (rand(T, 2).*x₀)..., x₀; kwargs...)

muller(f, xᵢ₋₂, xᵢ₋₁, xᵢ; kwargs...) =
    muller(f, promote(xᵢ₋₂, xᵢ₋₁, xᵢ)...; kwargs...)

function muller(f, oldest::T, older::T, old::T;
    xatol=nothing, xrtol=nothing, maxevals=300) where {T}
    @assert old ≠ older ≠ oldest ≠ old # we want q to be non-degenerate
    xᵢ₋₂, xᵢ₋₁, xᵢ = oldest, older, old
    fxᵢ₋₂, fxᵢ₋₁ = f(xᵢ₋₂), f(xᵢ₋₁)

    RT = typeof(abs(oldest))
    atol = xatol !== nothing ? xatol : oneunit(RT) * (eps(one(RT)))^(4/5)
    rtol = xrtol !== nothing ? xrtol : eps(one(RT))^(4/5)

    for i in 1:maxevals÷3
        # one evaluation per iteration
        fxᵢ = f(xᵢ)
        x = muller_step(xᵢ₋₂, xᵢ₋₁, xᵢ, fxᵢ₋₂, fxᵢ₋₁, fxᵢ)

        if isnan(x)
            @warn "The algorithm might not have converged, stopping at i=$i:" abs(xᵢ - xᵢ₋₁)
            return xᵢ
        end

        # @debug "Iteration $i:" xᵢ₋₂ xᵢ₋₁ xᵢ x abs(x-xᵢ)
        xᵢ₋₂, xᵢ₋₁, xᵢ = xᵢ₋₁, xᵢ, x
        fxᵢ₋₂, fxᵢ₋₁ = fxᵢ₋₁, fxᵢ
        #stopping criterion
        isapprox(xᵢ, xᵢ₋₁, atol=atol, rtol=rtol) && return xᵢ
    end
    @warn "The algorithm might not have converged, maxevals limit hit:" abs(xᵢ₋₁- xᵢ)
    return xᵢ
end

function muller_step(a,b,c,fa,fb,fc)
    a,b,c = promote(a,b,c)
    q = qq(a, b, c)
    q² = q^2
    q1 = q+one(q)

    A =      q*fc - q*q1*fb + q²*fa
    B = (q1+q)*fc - q1^2*fb + q²*fa
    C =     q1*fc

    den = let
        Δ = B^2 - 4A*C
        typeof(Δ) <: Real && Δ < 0 && throw(
            DomainError(Δ, "Discriminant is negative and the function most likely has complex roots. You might want to call muller with complex input."))
        Δ = √Δ
        d⁺ = B + Δ
        d⁻ = B - Δ
        abs(d⁺) > abs(d⁻) ? d⁺ : d⁻
    end
    return c - (c - b)*2C/den
end

@inline qq(a, b, c) = (c - b)/(b - a)

"""
    newton((f, f'), x0; xatol=nothing, xrtol=nothing, maxevals=100)
    newton(fΔf, x0; xatol=nothing, xrtol=nothing, maxevals=100)

Newton's method.

Function may be passed in as a tuple (f, f') *or* as function which returns (f,f/f').

Examples:
```
newton((sin, cos), 3.0)
newton(x -> (sin(x), sin(x)/cos(x)), 3.0, xatol=1e-10, xrtol=1e-10)
```

Note: unlike the call `newton(f, fp, x0)`--which dispatches to a method of `find_zero`, these
two interfaces will specialize on the function that is passed in. This means, these functions
will be faster for subsequent calls, but may be slower for an initial call.

Convergence here is decided by x_n ≈ x_{n-1} using the tolerances specified, which both default to
`eps(T)^4/5` in the appropriate units.

"""
struct TupleWrapper{F, Fp}
f::F
fp::Fp
end
(F::TupleWrapper)(x) = begin
    u, v = F.f(x), F.fp(x)
    return (u, u/v)
end

newton(f::Tuple, x0; kwargs...) = newton(TupleWrapper(f[1],f[2]), x0; kwargs...)
function newton(f, x0; xatol=nothing, xrtol=nothing, maxevals = 100)

    x = float(x0)
    T = typeof(x)
    atol = xatol != nothing ? xatol : oneunit(T) * (eps(one(T)))^(4/5)
    rtol = xrtol != nothing ? xrtol : eps(one(T))^(4/5)

    xo = Inf
    for i in 1:maxevals

        fx, Δx = f(x)
        iszero(fx) && return x

        x -= Δx

        if isapprox(x, xo, atol=atol, rtol=rtol)
            return x
        end

        xo = x
    end

    error("No convergence")
end

## This is basically Order0(), but with different, default, tolerances employed
## It takes more function calls, but works harder to find exact zeros
## where exact means either iszero(fx), adjacent floats have sign change, or
## abs(fxn) <= 8 eps(xn)
"""
    dfree(f, xs)

A more robust secant method implementation

Solve for `f(x) = 0` using an alogorithm from *Personal Calculator Has Key
to Solve Any Equation f(x) = 0*, the SOLVE button from the
[HP-34C](http://www.hpl.hp.com/hpjournal/pdfs/IssuePDFs/1979-12.pdf).

This is also implemented as the `Order0` method for `find_zero`.

The inital values can be specified as a pair of two values, as in
`(a,b)` or `[a,b]`, or as a single value, in which case a value of `b`
is computed, possibly from `fb`.  The basic idea is to follow the
secant method to convergence unless:

* a bracket is found, in which case bisection is used;

* the secant method is not converging, in which case a few steps of a
  quadratic method are used to see if that improves matters.

Convergence occurs when `f(m) == 0`, there is a sign change between
`m` and an adjacent floating point value, or `f(m) <= 2^3*eps(m)`.

A value of `NaN` is returned if the algorithm takes too many steps
before identifying a zero.

# Examples

```julia
Roots.dfree(x -> x^5 - x - 1, 1.0)
```

"""
function dfree(f, xs)

    if length(xs) == 1
        a = float(xs[1])
        fa = f(a)

        h = eps(one(a))^(1/3)
        da = h*oneunit(a) + abs(a)*h^2 # adjust for if eps(a) > h
        b = float(a + da)
        fb = f(b)
    else
        a, b = promote(float(xs[1]), float(xs[2]))
        fa, fb = f(a), f(b)
    end

    nan = (0*a)/(0*a) # try to preserve type
    cnt, MAXCNT = 0, 5 * ceil(Int, -log(eps(one(a))))  # must be higher for BigFloat
    MAXQUAD = 3

    if abs(fa) > abs(fb)
        a,fa,b,fb=b,fb,a,fa
    end

    # we keep a, b, fa, fb, gamma, fgamma
    quad_ctr = 0
    while !iszero(fb)
        cnt += 1

        if sign(fa) * sign(fb) < 0
            return bisection(f, a, b)
        end

        # take a secant step
        gamma =  float(b - (b-a) * fb / (fb - fa))
        # modify if gamma is too small or too big
        if iszero(abs(gamma-b))
            gamma = b + 1/1000 * abs(b-a)  # too small
        elseif abs(gamma-b)  >= 100 * abs(b-a)
            gamma = b + sign(gamma-b) * 100 * abs(b-a)  ## too big
        end
        fgamma = f(gamma)

        # change sign
        if sign(fgamma) * sign(fb) < 0
            return bisection(f, gamma, b)
        end

        # decreasing
        if abs(fgamma) < abs(fb)
            a,fa, b,fb = b, fb, gamma, fgamma
            quad_ctr = 0
            cnt < MAXCNT && continue
        end

        gamma = float(quad_vertex(a,fa,b,fb,gamma,fgamma))
        fgamma = f(gamma)
        # decreasing now?
        if abs(fgamma) < abs(fb)
            a,fa, b,fb = b, fb, gamma, fgamma
            quad_ctr = 0
            cnt < MAXCNT && continue
        end

        quad_ctr += 1
        if (quad_ctr > MAXQUAD) || (cnt > MAXCNT) || iszero(gamma - b)  || isnan(gamma)
            bprev, bnext = prevfloat(b), nextfloat(b)
            fbprev, fbnext = f(bprev), f(bnext)
            sign(fb) * sign(fbprev) < 0 && return b
            sign(fb) * sign(fbnext) < 0 && return b
            for (u,fu) in ((b,fb), (bprev, fbprev), (bnext, fbnext))
                abs(fu)/oneunit(fu) <= 2^3*eps(u/oneunit(u)) && return u
            end
            return nan # Failed.
        end

        if abs(fgamma) < abs(fb)
            b,fb, a,fa = gamma, fgamma, b, fb
        else
            a, fa = gamma, fgamma
        end

    end
    b

end


export _build, _get_kernel, funcs, r
using Statistics
using LinearAlgebra
using StaticArrays
#########################################################
#             Radial Basis functions                    #
#########################################################

phi_MQ(r, e) = sqrt(1 + (e .* r) .^ 2)
phi_InvMQ(r, e) = phi_MQ(r, e)^-1
phi_Q(r, e) = phi_MQ(r, e)^-2
phi_Gauss(r, e) = exp(-(e .* r)^2)
phi_Multiquadric(r, e) = -sqrt(1 + r .^ 2)
phi_InvMultiquadric(r, e) = 1 / (-sqrt(1 + r .^ 2))
phi_thin_plate_spline(r, e) = r .^ 2 .* log(r + 1e-5)
phi_cubic(r, e) = (r + 1) .^ -3
phi_quintic(r, e) = (r + 1) .^ -5
funcs = Dict([("MQ", phi_MQ),
    ("InvMQ", phi_InvMQ),
    ("Quadratic", phi_Q),
    ("Gaussian", phi_Gauss),
    ("minusMQ", phi_Multiquadric),
    ("minusInvMQ", phi_InvMultiquadric),
    ("Cubic", phi_cubic),
    ("Quintic", phi_quintic),
    ("ThinPlate", phi_thin_plate_spline)
])

"""
The function to retrive the basis for interpolation
    -> Input:
        :basis: String with the name of the function
                being selected. The available options are:
                ["MQ" , "InvMQ" , "Quadratic , "Gaussian"]
    ->Output:
        :basis function: two float parameters=> e (the epsilopn of the fuction)
                                                r (Distance form center)

"""
function _get_kernel(basis::String = "MQ"; func_dict = funcs)
    if basis ∉ keys(func_dict)
        throw(DomainError(basis, "Basis function not implemented\n If you want your own function set it using φ parameter"))
    end
    func_dict[basis]
end

# Distance calculation function
r(x, xi) = norm(x .- xi)

"""
build the onterpolation matrix

-> Input:
        :basis: ::Matrix with data to be interpolated
->Output:
        :Weights vector: Vector containing the weights of the interpolation
"""
function _build(X, y; φ = _get_kernel["QM"], ϵ = 1.0)
    A = ones((size(X)[1], size(X)[1]))
    for i = 1:size(X)[1], j = 1:size(X)[1]
        A[i, j] = φ(r(X[i, :], X[j, :]), ϵ)
    end
    sigm_y = std(y)
    if std(y) != 0
        newy = (y .- mean(y)) / std(y)
        w = (pinv(A) * newy)
    else
        w = (pinv(A) * y)
    end
    w
end


# Methods to provide grids and distance-matrices for
# different topologies of Kohonen maps (SOMs).

"""
    gridRectangular(xdim, ydim)

Create coordinates of all neurons on a rectangular SOM.

The return-value is an array of size (Number-of-neurons, 2) with
x- and y- coordinates of the neurons in the first and second
column respectively.
The distance between neighbours is 1.0.
The point of origin is bottom-left.
The first neuron sits at (0,0).
"""
function gridRectangular(xdim, ydim)

    grid = zeros(Float64, (xdim*ydim, 2))
    for ix in 1:xdim
        for iy in 1:ydim

            grid[ix+(iy-1)*xdim, 1] = ix-1
            grid[ix+(iy-1)*xdim, 2] = iy-1
        end
    end
    return grid
end

"""
    gridHexagonal(xdim, ydim)

Create coordinates of all neurons on a hexagonal SOM.

The return-value is an array of size (Number-of-neurons, 2) with
x- and y- coordinates of the neurons in the first and second
column respectively.
The distance between neighbours is 1.0.
The point of origin is bottom-left.
The first neuron sits at (0,0).
"""
function gridHexagonal(xdim, ydim)

    grid = zeros(Float64, (xdim*ydim, 2))
    rowHeight = √(1-0.25)

    for ix in 1:xdim
        for iy in 1:ydim

            if iy % 2 != 0    # 1st, 3rd, ... line
                grid[ix+(iy-1)*xdim, 1] = ix - 1
            else                    # 2nd, 4th, ... line
                grid[ix+(iy-1)*xdim, 1] = ix - 0.5
            end

            grid[ix+(iy-1)*xdim, 2] = (iy-1) * rowHeight
        end
    end
    return grid
end

"""
    gridSpherical(numNeurons)

Create coordinates of dim neurons on a spherical SOM.

The neurons are almost equally distributed on a sphere by
applying Fibonacci's spirals.
The radius of the sphere is adapted to get a distance of neighbours
of about 1.0.
The return-value is an array of size (Number-of-neurons, 3) with
x-, y- and z- coordinates of the neurons in the first, second and third
column respectively.
The point of origin is bottom-left.
The center of the sphere sits at (0,0).
"""
function gridSpherical(numNeurons)

    r = √(numNeurons/(4 * π))
    grid = zeros(Float64, (numNeurons, 3))

    # iterate Fibonacci spiral:
    for i in 1:(numNeurons)
        py = (i-1) * 2/numNeurons - 1 + (1/numNeurons)
        pr = √(1-py^2)
        ϕ  = (i-1) * π * (3-√5)
        px = cos(ϕ) * pr
        pz = sin(ϕ) * pr

        #println("raw: $px $py $pz")

        # shift point from origin to atom location:
        px = r * px
        py = r * py
        pz = r * pz

        #println("scaled: $px $py $pz")
        grid[i,:] = [px, py, pz]
    end
    return grid
end

# verify sphere:
# x = gridSpherical(50)
# R"""
# library(rgl)
# plot3d($x)
# #readline(prompt="Press [enter] to continue")
# """

"""
    distMatrix(grid::Array, toroidal::Bool)

Return the distance matrix for a non-toroidal or toroidal SOM.

# Arguments
- `grid`: coordinates of all neurons as generated by
          one of the `grid-`functions with x-coordinates in 1st column
          and y-coordinates in 2nd column.
- `toroidal`: true for a toroidal SOM.
"""
function distMatrix(grid::Array, toroidal::Bool)

    X = 1
    Y = 2
    xdim = maximum(grid[:,X]) - minimum(grid[:,X]) + 1.0
    ydim = maximum(grid[:,Y]) - minimum(grid[:,Y]) + 1.0

    numNeurons = nrow(grid)

    dm = zeros(Float64, (numNeurons,numNeurons))
    for i in 1:numNeurons
        for j in 1:numNeurons
            Δx = abs(grid[i,X] - grid[j,X])
            Δy = abs(grid[i,Y] - grid[j,Y])

            if toroidal
                Δx = min(Δx, xdim-Δx)
                Δy = min(Δy, ydim-Δy)
            end

            dm[i,j] = √(Δx^2 + Δy^2)
        end
    end
    # show(STDOUT, "text/plain",  grid)
    # show(STDOUT, "text/plain",  dm)
    return dm
end

"""
    distMatrixSphere(grid::Array)

Return the distance matrix for a spherical SOM.

# Arguments
- `grid`: coordinates of all neurons as gereated by
          one of the `grid-`functions with x-coordinates in 1st,
          y-coordinates in 2nd and z-coordinates in 3rd column.
          Point of origin must be in the center of the spere.
"""
function distMatrixSphere(grid::Array)

    r = (maximum(grid) - minimum(grid)) / 2

    numNeurons = nrow(grid)

    dm = zeros(Float64, (numNeurons,numNeurons))
    for i in 1:numNeurons
        for j in 1:numNeurons
            len_i = norm(grid[i,:])
            len_j = norm(grid[j,:])
            bog = grid[i,:] ⋅ grid[j,:] / (len_i * len_j)
            if bog > 1
                bog = 1.0
            end
            α = acos(bog)
            dm[i,j] = r * α
        end
    end
    return dm
end


# Locally Linear Embedding (LLE)
# ------------------------
# Nonlinear dimensionality reduction by locally linear embedding,
# Roweis, S. & Saul, L., Science 290:2323 (2000)

#### LLE type
immutable LLE{T <: Real} <: SpectralResult
    k::Int
    λ::AbstractVector{T}
    proj::Projection{T}

    LLE{T}(k::Int, λ::AbstractVector{T}, proj::Projection{T}) = new(k, λ, proj)
end

## properties
outdim(M::LLE) = size(M.proj, 1)
projection(M::LLE) = M.proj

eigvals(M::LLE) = M.λ
neighbors(M::LLE) = M.k

## show & dump
function show(io::IO, M::LLE)
    print(io, "LLE(indim = $(indim(M)), outdim = $(outdim(M)), neighbors = $(neighbors(M)))")
end

function dump(io::IO, M::LLE)
    show(io, M)
    println(io, "eigenvalues: ")
    Base.showarray(io, M.λ', header=false, repr=false)
    println(io)
    println(io, "projection:")
    Base.showarray(io, M.proj, header=false, repr=false)
end

## interface functions
function transform{T<:Real}(::Type{LLE}, X::DenseMatrix{T}; d::Int=2, k::Int=12)
    n = size(X, 2)

    # Construct NN graph
    D, E = find_nn(X, k)

    # Select largest connected component
    cc = components(E)
    c = cc[indmax(map(size, cc))]
    if length(cc) == 1
        c = cc[1]
    else
        c = cc[indmax(map(size, cc))]
        # renumber edges
        R = Dict(c, 1:length(c))
        Ec = zeros(Int,k,length(c))
        for i = 1 : length(c)
            Ec[:,i] = map(i->get(R,i,0), E[:,c[i]])
        end
        E = Ec
        X = X[:,c]
    end

    if k > d
        warn("K>D: regularization will be used")
        tol = 1e-5
    else
        tol = 0
    end

    # Reconstruct weights
    W = zeros(k, n)
    for i = 1 : n
        Z = X[:, E[:,i]] .- X[:,i]
        C = Z'*Z
        C = C + eye(k, k) * tol * trace(C)
        wi = vec(C\ones(k, 1))
        W[:, i] = wi./sum(wi)
    end

    # Compute embedding
    M = speye(n,n)
    for i = 1 : n
        w = W[:, i]
        jj = E[:, i]
        M[i,jj] = M[i,jj] - w'
        M[jj,i] = M[jj,i] - w
        M[jj,jj] = M[jj,jj] + w*w'
    end

    λ, V = decompose(M, d)
    return LLE{T}(k, λ, scale!(V', sqrt(n)))
end


#=
  A/B test

  From http://rpubs.com/rasmusab/exercise_2_bayesian_ab_testing
  """
  Exercise 2: Bayesian A/B testing for Swedish Fish Incorporated with Stan
  Rasmus Bååth

  Swedish Fish Incorporated is the largest Swedish company delivering fish by mail order,
  but you probably already knew that. The marketing department have done a pilot study and
  tried two different marketing methods:

  A: Sending a mail with a colorful brochure that invites people to sign up for a one year
     salmon subscription.

  B: Sending a colorful brochure that invites people to sign up for a one year
     salmon subscription nd that includes a free salmon.

  The marketing department sent out 16 mails of type A and 16 mails of type B.
  Six Danes that received a mail of type A signed up for one year of salmon, and
  ten Danes that received a mail of type B signed up!

  The marketing department now wants to know, which method should we use, A or B?

  At the bottom of this document you’ll find a solution. But try yourself first!
  Question I: Build a Bayesian model in Stan that answers the question: What is the
  probability that method B is better than method A?
  """

  See ~/stan/ab_testing.R
      ~/blog/ab_testing.blog
      ~/psi/ab_testing.psi
      ~/webppl/ab_testing.wppl
=#

using Turing, StatsPlots, DataFrames
# using ReverseDiff, Zygote, Tracker
# Turing.setadbackend(:reversediff)
# Turing.setadbackend(:zygote)
# Turing.setadbackend(:tracker)

include("jl_utils.jl")

#=
ummary Statistics
  parameters      mean       std   naive_se      mcse          ess      rhat
      Symbol   Float64   Float64    Float64   Float64      Float64   Float64

       rateA    0.3893    0.1126     0.0003    0.0009   15774.8462    1.0004
       rateB    0.6121    0.1108     0.0003    0.0008   16943.7123    1.0001

Quantiles
  parameters      2.5%     25.0%     50.0%     75.0%     97.5%
      Symbol   Float64   Float64   Float64   Float64   Float64

       rateA    0.1839    0.3088    0.3857    0.4659    0.6183
       rateB    0.3847    0.5397    0.6153    0.6918    0.8148

mean(rateB > rateA): 0.91461875
mean(rateB - rateA): 0.22276908208667892
=#
@model function ab_testing(nA=16,nB=16,obsSA=6, obsSB=10)
    # nA = 16 # Number of sent mail
    # nB = 16

    # obsSA = 6; # Number of signments (observed)
    # obsSB = 10

    rateA ~ Beta(1,1) # priors
    rateB ~ Beta(1,1)

    obsSA ~ Binomial(nA,rateA) # likelihood
    obsSB ~ Binomial(nB,rateB)

    # This variant don't work!
    # sA ~ Binomial(nA,rateA) # likelihood
    # sB ~ Binomial(nB,rateB)
    # sA == obsSA || begin Turing.@addlogprob! -Inf; return; end
    # sB == obsSB || begin Turing.@addlogprob! -Inf; return; end

end

model = ab_testing(16,16,6,10)
# model = ab_testing(160,160,60,100)
num_chains = 4
# chains = sample(model, Prior(), MCMCThreads(), 1000, num_chains)
chains = sample(model, MH(), MCMCThreads(), 40_000, num_chains)
# chains = sample(model, MH(), 40_000)

# chains = sample(model, PG(20), MCMCThreads(), 10_000, num_chains)
# chains = sample(model, IS(), MCMCThreads(), 10_000, num_chains)

# chains = sample(model, Gibbs(MH(:sA,:sB),NUTS(1000,0.65,:rateA,:rateB)), MCMCThreads(), 40_000, num_chains)

display(chains)
# display(plot(chains))

df = DataFrame(chains);
println("mean(rateB > rateA): ", mean(df[!,"rateB"] .> df[!,"rateA"]))
println("mean(rateB - rateA): ", mean(df[!,"rateB"] .- df[!,"rateA"]))


# Benchmarking the NLsolve.fixedpoint() function in the use case of: https://lectures.quantecon.org/jl/mccall_model_with_separation.html

# Dependencies.
using Distributions # So that some method overwritten error from NLsolve doesn't intermingle with the benchmarks.

# Wages (global since used in constructor)
const n = 60                                   # n possible outcomes for wage
const default_w_vec = linspace(10, 20, n)      # wages between 10 and 20
const a, b = 600, 400                          # shape parameters
const dist = BetaBinomial(n-1, a, b)
const default_p_vec = pdf.(dist, support(dist))

# Model Struct (global since types can't be defined in local scope)
mutable struct McCallModel{TF <: AbstractFloat,
                        TAV <: AbstractVector{TF},
                        TAV2 <: AbstractVector{TF}}
    α::TF         # Job separation rate
    β::TF         # Discount rate
    γ::TF         # Job offer rate
    c::TF         # Unemployment compensation
    σ::TF         # Utility parameter
    w_vec::TAV    # Possible wage values
    p_vec::TAV2   # Probabilities over w_vec

    McCallModel(α::TF=0.2,
                β::TF=0.98,
                γ::TF=0.7,
                c::TF=6.0,
                σ::TF=2.0,
                w_vec::TAV=default_w_vec,
                p_vec::TAV2=default_p_vec) where {TF, TAV, TAV2} =
        new{TF, TAV, TAV2}(α, β, γ, c, σ, w_vec, p_vec)
end

# Implementation from McCall with separation lecture on QuantEcon
function default_quantecon()
    # Utility function.
    function u(c::Real, σ::Real)
        if c > 0
            return (c^(1 - σ) - 1) / (1 - σ)
        else
            return -10e6
        end
    end
    # Instantiate model object.
    mcm = McCallModel()
    # Define the function to be used in the iteration loop.
    """
    A function to update the Bellman equations.  Note that V_new is modified in
    place (i.e, modified by this function).  The new value of U is returned.
    """
    function update_bellman!(mcm::McCallModel,
                            V::AbstractVector,
                            V_new::AbstractVector,
                            U::Real)
        # Simplify notation
        α, β, σ, c, γ = mcm.α, mcm.β, mcm.σ, mcm.c, mcm.γ

        for (w_idx, w) in enumerate(mcm.w_vec)
            # w_idx indexes the vector of possible wages
            V_new[w_idx] = u(w, σ) + β * ((1 - α) * V[w_idx] + α * U)
        end

        U_new = u(c, σ) + β * (1 - γ) * U +
                        β * γ * dot(max.(U, V), mcm.p_vec)

        return U_new
    end
    # Define a function to carry out the iteration loop.
    function solve_mccall_model(mcm::McCallModel;
                                tol::AbstractFloat=1e-5,
                                max_iter::Integer=2000)

        V = ones(length(mcm.w_vec))    # Initial guess of V
        V_new = similar(V)             # To store updates to V
        U = 1.0                        # Initial guess of U
        i = 0
        error = tol + 1

        while error > tol && i < max_iter
            U_new = update_bellman!(mcm, V, V_new, U)
            error_1 = maximum(abs, V_new - V)
            error_2 = abs(U_new - U)
            error = max(error_1, error_2)
            V[:] = V_new
            U = U_new
            i += 1
        end

        return V, U
    end
    # Get results.
    return solve_mccall_model(mcm)
end

# Implementation of the above, but with one vector that holds the state of the calculation [U V]
function stacked_quantecon()
    # Utility function.
    function u(c::Real, σ::Real)
        if c > 0
            return (c^(1 - σ) - 1) / (1 - σ)
        else
            return -10e6
        end
    end
    # Instantiate model object.
    mcm = McCallModel()
    # Function for each loop of the iteration.
    """
    An (in-place) stacked implementation of the Bellman operator for the McCall model with separation. By convention, U is at the top of the vector.
    bellman_operator!(newVec::AbstractVector, oldVec::AbstractVector, model::McCallModel = mcm)
    """
    function bellman_operator!(newVec, oldVec, model = mcm)
        # Unpack parameters.
        α, β, σ, c, γ = mcm.α, mcm.β, mcm.σ, mcm.c, mcm.γ
        # Add new V(w) values to newVec.
        for (w_idx, w) in enumerate(mcm.w_vec)
            # w_idx indexes the vector of possible wages
            newVec[w_idx + 1] = u(w, σ) + β * ((1 - α) * oldVec[1 + w_idx] + α * oldVec[1])
        end
        # Add new U value to newVec.
        newVec[1] = u(c, σ) + β * (1 - γ) * oldVec[1] + β * γ * dot(max.(oldVec[1], oldVec[2:end]), mcm.p_vec)
    end
    # Function to carry out the iteration.
    """
    Modified version of solve_mccall_model(), which uses our bellman_operator! function.
    solve_mccall_model_inplace(mcm::McCallModel; tol::AbstractFloat=1e-5, max_iter::Integer=2000)
    """
    function solve_mccall_model(mcm; tol=1e-5, max_iter=2000)
        oldVec = ones(length(mcm.w_vec) + 1)  # Initial guess for states. Same as those in the QE implementation.
        newVec = similar(oldVec)  # To store updates.
        iter = 0    # Initialize the iteration counter.
        error = tol + 1 # Initialize the error.

        while error > tol && iter < max_iter
            bellman_operator!(newVec, oldVec, mcm)
            error = maximum(abs.(newVec - oldVec))
            iter += 1
            oldVec .= newVec
        end

        return newVec
    end
    # Get results.
    return solve_mccall_model(mcm)
end

# Implementation of stacked_quantecon(), but using the NLsolve fixed point method. m = 0.
function fixedpoint_iteration()
    # Utility function.
    function u(c::Real, σ::Real)
        if c > 0
            return (c^(1 - σ) - 1) / (1 - σ)
        else
            return -10e6
        end
    end
    # Instantiate model object.
    mcm = McCallModel()
    # Function for each loop of the iteration.
    """
    An (in-place) stacked implementation of the Bellman operator for the McCall model with separation. By convention, U is at the top of the state vector.
    bellman_operator!(newVec::AbstractVector, oldVec::AbstractVector, model::McCallModel = mcm)
    """
    function bellman_operator!(newVec, oldVec, model = mcm)
        # Unpack parameters.
        α, β, σ, c, γ = mcm.α, mcm.β, mcm.σ, mcm.c, mcm.γ
        # Add new V(w) values to newVec.
        for (w_idx, w) in enumerate(mcm.w_vec)
            # w_idx indexes the vector of possible wages
            newVec[w_idx + 1] = u(w, σ) + β * ((1 - α) * oldVec[1 + w_idx] + α * oldVec[1])
        end
        # Add new U value to newVec.
        newVec[1] = u(c, σ) + β * (1 - γ) * oldVec[1] + β * γ * dot(max.(oldVec[1], oldVec[2:end]), mcm.p_vec)
    end
    # Solve
    init = ones(length(mcm.w_vec)+1)
    return fixedpoint(bellman_operator!, init; ftol = 1e-5, iterations = 2000)
end

# Using fixedpoint() for variable m.
function fixedpoint_anderson_m(m)
    # Utility function.
    function u(c::Real, σ::Real)
        if c > 0
            return (c^(1 - σ) - 1) / (1 - σ)
        else
            return -10e6
        end
    end
    # Instantiate model object.
    mcm = McCallModel()
    # Function for each loop of the iteration.
    """
    An (in-place) stacked implementation of the Bellman operator for the McCall model with separation. By convention, U is at the top of the state vector.
    bellman_operator!(newVec::AbstractVector, oldVec::AbstractVector, model::McCallModel = mcm)
    """
    function bellman_operator!(newVec, oldVec, model = mcm)
        # Unpack parameters.
        α, β, σ, c, γ = mcm.α, mcm.β, mcm.σ, mcm.c, mcm.γ
        # Add new V(w) values to newVec.
        for (w_idx, w) in enumerate(mcm.w_vec)
            # w_idx indexes the vector of possible wages
            newVec[w_idx + 1] = u(w, σ) + β * ((1 - α) * oldVec[1 + w_idx] + α * oldVec[1])
        end
        # Add new U value to newVec.
        newVec[1] = u(c, σ) + β * (1 - γ) * oldVec[1] + β * γ * dot(max.(oldVec[1], oldVec[2:end]), mcm.p_vec)
    end
    # Solve
    init = ones(length(mcm.w_vec)+1)
    return fixedpoint(bellman_operator!, init; m = m, ftol = 1e-5, iterations = 2000)
end

function fixedpoint_newton()
    # Utility function.
    function u(c::Real, σ::Real)
        if c > 0
            return (c^(1 - σ) - 1) / (1 - σ)
        else
            return -10e6
        end
    end
    # Instantiate model object.
    mcm = McCallModel()
    # Function for each loop of the iteration.
    """
    An (in-place) stacked implementation of the Bellman operator for the McCall model with separation. By convention, U is at the top of the state vector.
    bellman_operator!(newVec::AbstractVector, oldVec::AbstractVector, model::McCallModel = mcm)
    """
    function bellman_operator!(newVec, oldVec, model = mcm)
        # Unpack parameters.
        α, β, σ, c, γ = mcm.α, mcm.β, mcm.σ, mcm.c, mcm.γ
        # Add new V(w) values to newVec.
        for (w_idx, w) in enumerate(mcm.w_vec)
            # w_idx indexes the vector of possible wages
            newVec[w_idx + 1] = u(w, σ) + β * ((1 - α) * oldVec[1 + w_idx] + α * oldVec[1])
        end
        # Add new U value to newVec.
        newVec[1] = u(c, σ) + β * (1 - γ) * oldVec[1] + β * γ * dot(max.(oldVec[1], oldVec[2:end]), mcm.p_vec)
    end
    # Solve
    init = ones(length(mcm.w_vec)+1)
    return fixedpoint(bellman_operator!, init; method = :newton, ftol = 1e-5, iterations = 2000)
end


A = [1 2 3; -4 -5 -6]

A[2,3]

length(A)

size(A,1)

size(A,2)

sumA = 0
for i = 1:size(A,1)
    for j = 1:size(A,2)
        sumA += A[i,j]
    end
end

println("Sum of the elements in A = ", sumA)

sumA = 0
for i = 1:size(A,1), j = 1:size(A,2)
    sumA += A[i,j]
end

println("Sum of the elements in A = ", sumA)

sumAsquared = 0
for a in A
    sumAsquared += a^2
end

print("Sum of the squares of the elements in A = ", sumAsquared)

C = ones(5,3)               # 5-by-4 matrix with all elements == 1

ones(5)      # 1-d array (can be used as a column vector)

ones(5,1)    # 2-d array, column vector

ones(1,5)    # 2-d array, row vector

D = [A; C]                     # Vertical concatenation - arrays must have same # columns
E = [zeros(5) C]               # Horizontal concatenation - arrays must have same # rows
F = [A ones(2,2); zeros(1,5)]  # General vertical-horizontal concatenation - all dimensions must match

G = [1:5 ones(Int64,5) 101:105]

B = A.^2 .- 3A           # Apply function elementwise

B = @. A^2 - 3A          # Easier syntax - entire expression evaluated elementwise

A[1,:]             # Row 1

A[:,1]             # Column 1

A[:,2:3]           # Columns 2-3

A[:,[1,3]] .= 0          # Set columns 1 and 3 to zero
A

A[:,2:3] = 2A[:,1:2]     # Set columns 2 and 3 to twice of columns 1 and 2
A


using InstantiateFromURL
# optionally add arguments to force installation: instantiate = true, precompile = true
github_project("QuantEcon/quantecon-notebooks-julia", version = "0.8.0")

using LinearAlgebra, Statistics
using Plots, Distributions, Random, Statistics
gr(fmt = :png, size = (900, 500))

function ksl(distribution, n = 100)
    title = nameof(typeof(distribution))
    observations = rand(distribution, n)
    sample_means = cumsum(observations) ./ (1:n)
    μ = mean(distribution)
    plot(repeat((1:n)', 2),
         [zeros(1, n); observations'], label = "", color = :grey, alpha = 0.5)
    plot!(1:n, observations, color = :grey, markershape = :circle,
          alpha = 0.5, label = "", linewidth = 0)
    hline!([μ], color = :black, linewidth = 1.5, linestyle = :dash, grid = false, label = ["Mean"])
    plot!(1:n, sample_means, linewidth = 3, alpha = 0.6, color = :green,
          label = "Sample mean")
    return plot!(title = title)
end

# Example distributions
distributions = [TDist(10), Beta(2,2), Gamma(5,2), Poisson(4), LogNormal(0.5), Exponential(1)]

Random.seed!(0); #reproducible results
ksl(Normal())
plot(ksl.(sample(distributions, 3, replace = false))..., layout = (3, 1), legend = false)

# Cauchy distribution may not converge as E|X| < ∞ not satisfied (heavy tailed)

function plotmeans(distribution::Distribution{Univariate,Continuous};n=1000)
    sample_mean  = cumsum(rand(distribution,n)) ./ (1:n)
    plot(1:n, sample_mean, label = "sample mean")
    hline!([0], color = :black, linestyle = :dash)
end

Random.seed!(0);
plotmeans(Cauchy(),n=10000)

using StatsPlots

function simulation1(distribution::Distribution{Univariate,Continuous}; n=250, k = 10_000)
    σ = std(distribution)
    y = rand(distribution,n,k)
    y .-= mean(distribution)
    y = mean(y,dims=1)
    y = √n *vec(y)
    density(y,label = "Empirical Distribution")
    return plot!(Normal(0,σ), linestyle = :dash, label = "Normal(0.0, $(σ^2))")
end

Random.seed!(0);
simulation1(Exponential(0.5), n=100)

function simulation2(distribution = Beta(2, 2), n = 5, k = 10_000)
    y = rand(distribution, k, n)
    for col in 1:n
        y[:,col] += rand([-0.5, 0.6, -1.1], k)
    end
    y = (y .- mean(distribution)) ./ std(distribution)
    y = cumsum(y, dims = 2) ./ sqrt.(1:5)' # return grid of data
end

ys = simulation2()
plots = [] # would preallocate in optimized code
for i in 1:size(ys, 2)
    p = density(ys[:, i], linealpha = i, title = "n = $i")
    push!(plots, p)
end

plot(plots..., legend = false)

# Exercise 1

function exercise1(a,b; g = sin, g′ = cos, n=250, k=10_000)
    distribution = Uniform(a,b)
    μ, σ = mean(distribution), std(distribution)
    gμ = sin.(mean(distribution))
    X = rand(distribution,n,k)
    X = mean(X, dims = 1)
    X = vec(X)
    y = √n .* vec(g.(X) .- g.(μ))
    density(y,label = "Empirical Distribution")
    return plot!(Normal(0,g′(μ).*σ), linestyle = :dash, label = "Asymptotic")
end

exercise1(0.0,π/2,k=15000)

# Exercise 2
function exercise2(;n = 250, k = 50_000, dw = Uniform(-1, 1), du = Uniform(-2, 2))
    vw = var(dw)
    vu = var(du)
    Σ = [vw vw; vw vu+vw]
    Q = inv(sqrt(Σ))

    function generate_data(dw, du, n)
        dw = rand(dw, n)
        X = [dw dw + rand(du, n)]
        return sqrt(n) * mean(X, dims = 1)
    end

    X = mapreduce(x -> generate_data(dw, du, n), vcat, 1:k)
    X = sqrt(n) .* mean(X, dims=1)
    X = Q*X'
    X = sum(abs2, X, dims = 1)
    X = vec(X)
    density(X, label = "", xlim = (0, 10))
    return plot!(Chisq(2), color = :black, linestyle = :dash,
                 label = "Chi-squared with 2 degrees of freedom", grid = false)
end
exercise2()