using EngEconomics, Roots, Plots

# Given
# Job1 = a
# Job2 = b
# Job3 = c
aInit = 100000
aAnnual = -75000
aFinal = 200000

bInit = 150000
bAnnual = -100000
bFinal = 230000

cInit = 175000
cAnnual = -150000
cFinal = 300000

MARR = 0.1
N = 3

# Find Best alternative from IRR analysis
# Step 1: Sort from smallest to biggest initial
# - X, c, b, a (where X is do nothing)
ΔCX_Init = cInit
ΔCX_Annual = cAnnual
ΔCX_Final = cFinal
ΔCX_IRR_eq(x) = ΔCX_Init * compoundAmountFactor(MARR, N) + ΔCX_Annual * uniformSeriesCompoundAmoundFactor(x, N) + ΔCX_Final

xVec = collect(0:0.01:1)
yVec = zeros(size(xVec))
plot(xVec, ΔCX_IRR_eq.(xVec))
plot!(xVec, yVec)

ΔCX_IRR = find_zero(ΔCX_IRR_eq, MARR)

if ΔCX_IRR >= MARR
	println("C")
else
	println("Do Nothing")
end

# BC
ΔBC_Init = bInit - cInit
ΔBC_Annual = bAnnual - cAnnual
ΔBC_Final = bFinal - cFinal
ΔBC_IRR_eq(x) = ΔBC_Init * compoundAmountFactor(x, N) + ΔBC_Annual * uniformSeriesCompoundAmoundFactor(MARR, N) + ΔBC_Final

plot(xVec, ΔBC_IRR_eq.(xVec))
plot!(xVec, yVec)

ΔBC_IRR = find_zero(ΔBC_IRR_eq, MARR)

if ΔBC_IRR >= MARR
	println("B")
else
	println("C")
end

# AB
ΔAB_Init = aInit - bInit
ΔAB_Annual1 = aAnnual - bAnnual
ΔAB_Final = aFinal - bFinal
ΔAB_IRR_eq(x) = ΔAB_Init * compoundAmountFactor(x, N) + ΔAB_Annual1 * uniformSeriesCompoundAmoundFactor(MARR, N) + ΔAB_Final

plot(xVec, ΔAB_IRR_eq.(xVec))
plot!(xVec, yVec)

ΔAB_IRR = find_zero(ΔAB_IRR_eq, MARR)

if ΔAB_IRR >= MARR
	println("A")
else
	println("B")
end

# If non-uniform cash flows, then use ERR
# - ERR means that if it's not an investment use the MARR rather than x
# - what this really means is, if \Delta_Job1_Job2_THINGY is negative, use x, because this is an investment so we use x
# - if \Delta_[]_[] is positive, then it is NOT an investment, so we ought to use MARR rather than x


"""
    make_sin(n, start, stop; noise = 0.3, f_rand = randn) -> x, y

Generates `n` noisy equally spaces samples of a sinus from `start` to `stop`
by adding `noise .* f_rand(length(x))` to the result of `fun(x)`.
"""
function make_sin(n::Int = 50, start::Real = 0, stop::Real = 2π; noise::Real = 0.3, f_rand::Function = randn)
    x = collect(range(start, stop=stop, length=n))
    y = sin.(x) .+ noise  .* f_rand.()
    return x, y
end

"""
    make_poly(coef, x; noise = 0.01, f_rand = randn) -> x, y

Generates a noisy response for a polynomial of degree `length(coef)`
using the vector `x` as input and adding `noise .* f_randn(length(x))` to the result.
The vector `coef` contains the coefficients for the terms of the polynome.
The first element of `coef` denotes the coefficient for the term with
the highest degree, while the last element of `coef` denotes the intercept.
"""
function make_poly(coef::AbstractVector{R}, x::AbstractVector{T};
                noise::Real = 0.1, f_rand::Function = randn) where {T<:Real,R<:Real}
    n = length(x)
    m = length(coef)
    x_vec = collect(x)
    y = zeros(n)
    @inbounds for i = 1:n
        for k = 1:m
            y[i] += coef[k] * x_vec[i]^(m-k)
        end
    end
    y .+= noise .* f_rand.()
    x_vec, y
end

"""
    make_spiral(n, a, theta, b; noise = 0.01, f_rand = randn) -> x, y

Generates `n` noisy responses for a spiral with two labels. Uses the radius, angle
and scaling arguments to space the points in 2D space and adding `noise .* f_randn(n)`
to the response.
"""
function make_spiral(n::Int = 97, a::Real = 6.5, theta::Real = 16.0, b::Real=104.0;
                noise::Real = 0.1, f_rand::Function = randn)
    x = zeros(Float64, (2, 2*n))
    y = zeros(Int, 2*n)
    index = 0:1.0:(n-1)
    for i = 1:n
        _angle = index[i]*pi/theta
    	_radius = a * (b-index[i]) / b
    	x_coord = _radius * sin(_angle)
    	y_coord = _radius * cos(_angle)
    	x[1, i] = x_coord
    	x[2, i] = y_coord
        x[1, n+i] = -(x_coord)
    	x[2, n+i] = -(y_coord)
        y[i] = 1
        y[n+i] = 0
    end
    x[1, :] .+= noise .* f_rand.()
    x[2, :] .+= noise .* f_rand.()
    x, y
end


using Distributions

"""
Example of a computation of beta credible interval with MC sampling.
"""
function mcBetaQuantileDemo()
    S = 47
    N = 100
    a = S+1
    b = (N-S)+1
    α = .05

    SS = 1000
    X = rand(Beta(a, b), SS)
    sort!(X)

    Xl = X[round(Int, SS*α/2)]
    Xu = X[round(Int, SS*(1-α/2))]
    CIhat = [Xl, Xu]

    CIhat2 = quantile(EmpiricalUnivariateDistribution(X), [α/2, 1-α/2])
    (CIhat, CIhat2)
end

"""
Example of a computation of normal credible interval with MC sampling.
"""
function mcNormalQuantileDemo()
    μ = 0
    Σ = 1
    S = 1000
    rn = Normal(μ, Σ)
    X = rand(rn, S)
    qs = [0.025 0.5 0.975]

    qexact = quantile(rn, qs)

    qapprox = quantile(EmpiricalUnivariateDistribution(X), qs)
end


#= -------------------------------------------------------------------------
# @file problem1.jl
#
# @date 12/22/15 21:56:50
# @author Martin Noblia
# @email martin.noblia@openmailbox.org
#
# @brief
#
# @detail
#
  Licence:
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License

---------------------------------------------------------------------------=#
# Enun
#=
If we list all the natural numbers below 10 that are multiples of 3 or 5,
we get 3, 5, 6 and 9. The sum of these multiples is 23.
Find the sum of all the multiples of 3 or 5 below 1000.
=#

"""
Function for collect the integers multiples of some integer
Input:
N(Int) Maximum integer
n(Int) integer multiple
Output:
collection of integers multiples of n
"""
mult_of(N::Int,n::Int) = find(collect(1:N) .% n .== 0)
s = sum(unique(vcat(mult_of(999, 3), mult_of(999, 5))));
print("The sum of all the multiples of 3 or 5 bellow 1000 is: $s")


# julia1_2.3.jl Primitive, Abstract and Composite Types
#
# Primitive Types
# Integer Types with maxnumber
println(Int8(127))
println(UInt8(255))
println(Int16(32767))
println(UInt16(65535))
println(Int32(2147483647))
println(UInt32(4294067295))
println(Int64(2)^63-1)
println(UInt64(2)^64-1)
#  Int is default for Int64 if a 64 bit CPU else 32
# UInt is default for UInt64 if a 64 bit CPU else 32
println(Int128(2)^127-1)
println(UInt128(2)^128-1)
println(BigInt(2)^2046-1)

# floating point Types and irrational numbers
# Float16 to Float64 and BigFloat
println(Float16(π))  # 11 bits of precision
println(Float32(π))  # 24 bits of precision
println(Float64(π))  # 53 bits of precision
println(BigFloat(π)) # 256 bits of precision - normally
precision(BigFloat)
println(π)           # irrational number cannot be represented as a rational
BigFloat(π)          #Bigfloat defaults to 256 binary places ~77 decimal places
BigFloat(π,500)      # 500+ bits of precision

# complex numbers
println(Complex{Float16}(π+2π*im)) # 11 bits precision each part
println(Complex{Float32}(π+2π*im)) # 24 bits precision each part
println(Complex{Float64}(π+2π*im)) # 53 bits precision each part
# full gamut of elementary functions work on complex numbers
# Complex shortcuts
ComplexF16(3.1+3.2im)  # Alias for Complex{Float16}()
ComplexF32(3.1+3.2im)  # Alias for Complex{Float32}()
ComplexF64(3.1+3.2im)  # Alias for Complex{Float64}()
Complex(3.1+3.2im)     #default for ComplexF64 if 64 bit CPU else ComplexF32

# Rational numbers
# used for exact arithmetic
# for example 1/3 cannot be represented exactly by any
#  floating point number.
# 1//3 represents the rational number 1/3 exactly
a = 1//3 + 2//5 - 3//7
b = (22//7)*(5//13)
c = ((22//7)/2)^2
print("a=$a, b=$b, c=$c")
Rational(3.875) # Make a rational fraction out of a Float
Rational(3.825) # a repeating binary pattern so not exact, use rationalize() function
BigFloat(3.825) # gives wrong precision due to prior conversion step
BigFloat(3825//1000) # or
BigFloat("3.825")


module Test_bouncingBall_with_measurement_analytic

using Measurements

const g  = 9.81
const e  = 0.7 ± 0.1
const h0 = 1.0 ± 0.2

#=
der(h) = v  ; h(0) = h0
der(v) = -g ; v(0) = 0

when h <= 0 then
	v = -e*v
end when

Analytic solution of differential equation with
h(t_ev) = h_ev, v(t_ev) = v_ev:

	v = -g*(t-t_ev) + v_ev
	h = -g/2*(t-t_ev)^2 + v_ev*(t-t_ev) + h_ev

First phase (t_ev = 0, v_ev=0, h_ev=h0)
	v = -g*t
	h = -g/2*t^2 + h0

First event:
	hev1 = 0 -> t_ev1 = sqrt(2*h0/g)
	            v_ev1_before = -g*t_ev1
				v_ev1_after  = -e*v_ev1_before

Second phase (t_ev = t_ev1, v_ev = v_ev1_after, h_ev = 0)
    v = -g*(t-t_ev1) + v_ev1_after
    h = -g/2*(t-t_ev1)^2 + v_ev1_after*(t-t_ev1)

Second event:
	h_ev2 = 0 = -g/2*(t-t_ev1)^2 + vev1_after*(t-tev_1)
	          = (t-t_ev1)*(-g/2*(t-t_ev1) + v_ev1_after)

	-> t_ev2-t_ev1 = 2*v_ev1_after/g

	v_ev2_before = -g*(t_ev2-t_ev1) + v_ev1_after
	             = -g*2*v_ev1_after/g + v_ev1_after
	   			 = -v_ev1_after
	v_ev2_after  = -e*v_ev2_before
=#

t_ev1        = sqrt(2*h0/g)
h_ev1        = -g/2*t_ev1^2 + h0
v_ev1_before = -g*t_ev1
v_ev1_after  = -e*v_ev1_before

t_ev2        = t_ev1 +2*v_ev1_after/g
h_ev2        = -g/2*(t_ev2-t_ev1)^2 + v_ev1_after*(t_ev2-t_ev1)
v_ev2_before = -v_ev1_after
v_ev2_after  = -e*v_ev2_before

@show t_ev1
@show h_ev1
@show v_ev1_before
@show v_ev1_after
@show t_ev2
@show h_ev2
@show v_ev2_before
@show v_ev2_after

# Alternative that computes t_ev1 with Roots.jl:

println("\n using Roots:")

using Roots
f(t)  = -g/2*t^2 + h0
t_ev1 = find_zero(f,(0.1±0.0, 0.5±0.0))
h_ev1 = f(t_ev1)
@show t_ev1, h_ev1
end


using SpecialFunctions, Plots, HCubature, LinearAlgebra, Random

function n_ball_volume_analytic(n_dims, r)
    V = π^(n_dims/2)*r^n_dims/gamma(n_dims/2 + 1)
    return V
end

function monte_carlo_sphere_test(n_samples, r, n_dims)
    #draw uniformly random points in n_space and see if distance to
    #origin is <= r
    Random.seed!(0)
    inside_ball = 0
    for i = 1 : n_samples
        if norm(rand(n_dims)) <= r
            inside_ball += 1
        end
    end
    ratio = inside_ball/n_samples
    volume = ratio*(2*r)^n_dims
    return volume
end

function cubature_sphere_volume(n_samples, r, n_dims)
    a = -ones(n_dims-1) .+ eps()
    b = ones(n_dims-1) .- eps()
    function f(x)
        inside_bit = r^2 - sum(x.^2)
        if inside_bit <= 0
            return 0.0
        else
            return sqrt(r^2 - sum(x.^2))
        end
    end
    I, e = hcubature(f, a, b, maxevals = n_samples)
    return 2*I
end

r = 1.0
n_dims = 2:15

V = n_ball_volume_analytic.(n_dims, r)

p1 = plot(n_dims, V, xlabel = "dimension", ylabel = "volume", label = "analytic")

n_samples = Int(1e6)

V_mc = monte_carlo_sphere_test.(n_samples, r, n_dims)

plot!(p1, n_dims, V_mc, xlabel = "dimension", ylabel = "volume", label = "mc")

V_cubature = cubature_sphere_volume.(n_samples, r, n_dims)

plot!(p1, n_dims, V_cubature, ylim = [0, 6], xlabel = "dimension", ylabel = "volume", label = "cubature")

p2 = plot(n_dims, abs.(V_mc .- V)./V, yaxis = :log, xlabel = "dimension", ylabel = "relative abs error", label = "mc")

plot!(p2, n_dims, abs.(V_cubature .- V)./V, yaxis = :log, xlabel = "dimension", ylabel = "relative abs error", label = "cubature")

plot(p1, p2)


# Load Julia packages (libraries) needed

using StatisticalRethinking, Optim
#gr(size=(600,600));

# ### snippet 2.6

# Grid of 1001 steps

p_grid = range(0, step=0.001, stop=1);

# all priors = 1.0

prior = ones(length(p_grid));

# Binomial pdf

likelihood = [pdf(Binomial(9, p), 6) for p in p_grid];

# As Uniform prior has been used, unstandardized posterior is equal to likelihood

posterior = likelihood .* prior;

# Scale posterior such that they become probabilities

posterior = posterior / sum(posterior);

# ### snippet 3.3

# Sample using the computed posterior values as weights

N = 10000
samples = sample(p_grid, Weights(posterior), N);

# In StatisticalRethinkingJulia samples will always be stored
# in an MCMCChains.Chains object.

chn = MCMCChains.Chains(reshape(samples, N, 1, 1), ["toss"]);

# Describe the chain

MCMCChains.describe(chn)

# Plot the chain

plot(chn)

# Compute the MAP (maximum_a_posteriori) estimate

x0 = [0.5]
lower = [0.0]
upper = [1.0]

function loglik(x)
  ll = 0.0
  ll += log.(pdf.(Beta(1, 1), x[1]))
  ll += sum(log.(pdf.(Binomial(9, x[1]), repeat([6], 1))))
  -ll
end

(qmap, opt) = quap(loglik, x0, lower, upper)

# Show optimization results

opt

# Fit quadratic approcimation

quapfit = [qmap[1], std(samples, mean=qmap[1])]

p = Vector{Plots.Plot{Plots.GRBackend}}(undef, 4)
p[1] = scatter(1:length(p_grid), samples, markersize = 2, ylim=(0.0, 1.3), lab="Draws")

# analytical calculation

w = 6
n = 9
x = 0:0.01:1
p[2] = plot( x, pdf.(Beta( w+1 , n-w+1 ) , x ), lab="Conjugate solution")
density!(p[2], samples, lab="Sample density")

# quadratic approximation

p[3] = plot( x, pdf.(Beta( w+1 , n-w+1 ) , x ), lab="Conjugate solution")
plot!( p[3], x, pdf.(Normal( quapfit[1], quapfit[2] ) , x ), lab="Quap approximation")

# ### snippet 2.7

w = 6; n = 9; x = 0:0.01:1
p[4] = plot( x, pdf.(Beta( w+1 , n-w+1 ) , x ), lab="Conjugate solution")
plot!(p[4], x, pdf.(Normal( 0.67 , 0.16 ) , x ), lab="Normal approximation")
plot(p..., layout=(2, 2))

# End of `02/clip-06-07.jl`


using FdeSolver
using Plots
using SpecialFunctions

## inputs
tSpan = [0, 1]     # [intial time, final time]
y0 = 0             # intial value
β = 0.9            # order of the derivative

# Equation
F(t, n, β, y) = (40320 ./ gamma(9 - β) .* t[n] .^ (8 - β) .- 3 .* gamma(5 + β / 2)
           ./ gamma(5 - β / 2) .* t[n] .^ (4 - β / 2) .+ 9/4 * gamma(β + 1) .+
           (3 / 2 .* t[n] .^ (β / 2) .- t[n] .^ 4) .^ 3 .- y[n] .^ (3 / 2))

# Jacobian
JacobF(t, n, β, y) = -(3 / 2) .* y[n] .^ (1 / 2)

## Numerical solution
t, Yapp = FDEsolver(F, tSpan, y0, β, nothing, StopIt = "Convergence", tol = 10e-8, itmax = 15)

t1, Yapp1 = FDEsolver(F, tSpan, y0, β, JacobF, StopIt = "Convergence", tol = 10e-8, itmax = 15)

#plot
plot(t, Yapp, linewidth = 5, title = "Solution of a 1D fractional IVP",
     xaxis = "Time (t)", yaxis = "y(t)", label = "Approximation")
plot!(t1, Yapp1, linewidth = 5, ls = :dot, label = "Approximation with Jacobian")
plot!(t, t -> (t.^8 - 3 * t .^ (4 + β / 2) + 9 / 4 * t.^β),
      lw = 3, ls = :dash, label = "Exact solution")


"""
    dircos(incl, decl)

Compute the three cartesian components of the unit vector aligned with a field vector defined by the two angles inclination `incl` and declination `decl`.

# Example
```julia-repl
julia> dircos(56.0, 0.0)
(0.5591929034707468, 0.0, 0.8290375725550417)
```
"""
function dircos(incl::T, decl::T) where T<:Real
    xincl = incl * pi / 180.0
    xdecl = decl * pi / 180.0
    a = cos(xincl) * cos(xdecl)
    b = cos(xincl) * sin(xdecl)
    c = sin(xincl)
    return a, b, c
end


using NumberIntervals, Turmeric, StaticArrays

rts = roots(sin, NumberInterval(-5, 5))

rts = roots(sin, NumberInterval(-5, 6), Bisection())
rts = roots(sin, NumberInterval(-5, 6), Newton())

# 2D:
f(x, y) = SVector(x^2 + y^2 - 1, y - x)
f(X) = f(X...)

rts = roots(f, SVector(NumberInterval(-5, 5), NumberInterval(-5, 5)))

# complex:
x = NumberInterval(-5, 6)
Xc = Complex(x, x)
f(z) = z^3 - 1

rts = roots(f, Xc, Bisection())
rts = roots(f, Xc, Newton())
rts = roots(f, Xc)

# From R docs:

# https://www.rdocumentation.org/packages/pracma/versions/1.9.9/topics/broyden

function g(x)
    (x1, x2, x3) = x
    SVector(    x1^2 + x2^2 + x3^2 - 1,
                x1^2 + x3^2 - 0.25,
                x1^2 + x2^2 - 4x3
            )
end

X = NumberInterval(-5, 5)
@time rts = roots(g, SVector(X, X, X))

h(xv) = ((x,y) = xv; SVector(2*x - y - exp(-x), -x + 2*y - exp(-y)))

rts = roots(h, SVector(X, X), Bisection())

# Dennis-Schnabel:
h(xv) = ((x, y) = xv; SVector(x^2 + y^2 - 2, exp(x - 1) + y^3 - 2))

rts = roots(h, SVector(X, X), Bisection())

# Test suites:

# http://folk.uib.no/ssu029/Pdf_file/Testproblems/testprobRheinboldt03.pdf

# http://www.mat.univie.ac.at/~neum/glopt/test.html

# http://titan.princeton.edu/TestProblems/

# http://www-sop.inria.fr/saga/POL/

##  MINPACK benchmarks: https://github.com/JuliaNLSolvers/NLsolve.jl/blob/master/test/minpack.jl

rosenbrock(x, y) = SVector( 1 - x, 10 * (y - x^2) )
rosenbrock(X) = rosenbrock(X...)

rosenbrock(xx) = ( (x, y) = xx; SVector( 1 - x, 1000 * (y - x^2) ) )
X = NumberInterval(-1e5, 1e5)

rts = roots(rosenbrock, SVector(X, X))

# and other files in NLsolve test suite

# Testing unconstrained optimization software, Moré, Garbow, Hillstrom ACM Trans. Math. Soft. 7 (1), 17-41 (1981)


#osaka.jl

"""
    Implementation of Osaka's extended treatment of Feynman's variational technique applied to the Polaron model; generalising it from the case at O^{∘}K to the case at finite temperature.

    See Osaka 1959:
    Progress of Theoretical Physics, Vol. 22, No.3, September 1959

    And Hellwarth 1999:
    https://doi.org/10.1103/PhysRevB.60.299

"""

using Optim
using QuadGK

# Equation 62b in Hellwarth. Equation 20 in Osaka.
A(v, w, β) = 3 / β * (log(v / w) - 1 / 2 * log(2 * π * β) - log(sinh(v * β / 2) / sinh(w * β / 2)))

# Equation 62d in Hellwarth.
Y(x, v, β) = 1 / (1 - exp(-v * β)) * (1 + exp(-v * β) - exp(-v * x) - exp(v * (x - β)))

# Osaka's version: Within Equation 16.
# Y(x, v, β) = 1 - exp(-v * x) + (1 - coth(v * β / 2)) * (cosh(v * x) - 1)

# Integrand of Equation 62c in Hellwarth.
B_integrand(x, v, w, β) =  (exp(β - x) + exp(x)) / sqrt(w^2 * x * (1 - x / β) + Y(x, v, β) * (v^2 - w^2) / v)

# Osaka's version: Within Equation 16.
# B_integrand(x, v, w, β) =  (exp(-x)) / sqrt(abs(w^2 * x * (1 - x / β) + Y(x, v, β) * (v^2 - w^2) / v))

# Equation 62c. NB: Extra exp(β) correction term from Osaka to correct Hellwarth typo.
B(v, w, β, α) = α * v / (sqrt(π) * (exp(β) - 1)) * QuadGK.quadgk(x -> B_integrand(x, v, w, β), 0, β / 2)[1]

# Osaka's version: Equation 16.
# B(v, w, β, α) = α * v * exp(β) / (sqrt(π) * (exp(β) - 1)) * QuadGK.quadgk(x -> B_integrand(x, v, w, β), 0, β)[1]

# Equation 62e in Hellwarth. Equation 17 in Osaka.
C(v, w, β) = 3 / 4 * (v^2 - w^2) / v * (coth(v * β / 2) - 2 / (v * β))

# Equation 62a in Hellwarth. In paragraph below Equation 22 in Osaka; has extra 1/β due to different definition of A, B & C.
osaka_free_energy(v, w, β, α) = -(A(v, w, β) + B(v, w, β, α) + C(v, w, β))


function extendedEuclideanAlgorithm(a, b)
    if b == 0
        return (a, 1, 0)
    end
    gcd, x1, y1 = extendedEuclideanAlgorithm(b, a % b)
    x = y1
    y = x1 - floor(a / b) * y1
    return gcd, x, y
end


function TT_SVD(tensor::Array{Float64},ϵ::Float64)
########################################################################
#   Computes the cores of a TT for the given tensor and accuracy (acc)
#   Resources:
#   V. Oseledets: Tensor-Train Decomposition, 2011, p.2301: Algorithm 1
#   April 2021, Clara Menzen
########################################################################
    N = ndims(tensor);
    cores = Vector{Array{Float64,3}}(undef,N);
    frobnorm = norm(tensor);

    δ = ϵ / sqrt(N-1) * frobnorm;
    err2 = 0;
    rprev = 1;
    sizes = size(tensor);
    C = reshape( tensor, (sizes[1], Int(length(tensor) / sizes[1]) ));
    for k = 1 : N-1
        # truncated svd
        F   = svd!(C);
        rcurr = length(F.S);

        sv2 = cumsum(reverse(F.S).^2);
        tr  = Int(findfirst(sv2 .> δ^2))-1;
        if tr > 0
            rcurr = length(F.S) - tr;
            err2 += sv2[tr];
        end

        # new core
        cores[k] = reshape(F.U[:,1:rcurr],(rprev,sizes[k],rcurr));
        rprev    = rcurr;
        C        = Diagonal(F.S[1:rcurr])*F.Vt[1:rcurr,:];
        C        = reshape(C,(rcurr*sizes[k+1], Int(length(C) / (rcurr*sizes[k+1])) ) );
    end
    cores[N] = reshape(C,(rprev,sizes[N],1));
    return MPT(cores), sqrt(err2)/frobnorm
end

function TT_SVD(tensor::Array{Float64},d::Int,ϵ::Float64)
    ########################################################################
    # for a symmetric matrix
        cores    = Vector{Array{Float64,3}}(undef,d);
        S        = zeros(2)
        frobnorm = norm(tensor);

        δ     = ϵ / sqrt(d-1) * frobnorm;
        err2  = 0;
        rprev = 1;
        sizes = size(tensor);
        C     = reshape( tensor, (sizes[1], Int(length(tensor) / sizes[1]) ));
        for k = 1 : d
            # truncated svd
            F   = svd!(C);
            rcurr = length(F.S);

            sv2 = cumsum(reverse(F.S).^2);
            tr  = Int(findfirst(sv2 .> δ^2))-1;
            if tr > 0
                rcurr = length(F.S) - tr;
                err2 += sv2[tr];
            end

            # new core

            cores[k] = reshape(F.U[:,1:rcurr],(rprev,sizes[k],rcurr));
            rprev    = rcurr;
            C        = Diagonal(F.S[1:rcurr])*F.Vt[1:rcurr,:];
            C        = reshape(C,(rcurr*sizes[k+1], Int(length(C) / (rcurr*sizes[k+1])) ) );
            S        = F.S[1:rcurr]
        end
        return MPT(cores), S, sqrt(err2)/frobnorm
    end


"""
    poissonfd(f,g,m,xspan,n,yspan)

Solve Poisson's equation on a rectangle by finite differences.
Function `f` is the forcing function and function `g` gives the
Dirichlet boundary condition. The rectangle is the tensor product of
intervals `xspan` and `yspan`,  and the discretization uses `m`+1
and `n`+1 points in the two coordinates.

Returns vectors defining the grid and a matrix of grid solution values.
"""
function poissonfd(f,g,m,xspan,n,yspan)
    # Discretize the domain.
    x,Dx,Dxx = FNC.diffmat2(m,xspan)
    y,Dy,Dyy = FNC.diffmat2(n,yspan)
    mtx = h -> [ h(x,y) for x in x, y in y ]
    X,Y = mtx((x,y)->x),mtx((x,y)->y)
    N = (m+1)*(n+1)   # total number of unknowns

    # Form the collocated PDE as a linear system.
    A = kron(I(n+1),sparse(Dxx)) + kron(sparse(Dyy),I(m+1))
    b = vec( mtx(f) )

    # Identify boundary locations.
    isboundary = trues(m+1,n+1)
    isboundary[2:m,2:n] .= false
    idx = vec(isboundary)

    # Apply Dirichlet condition.
    scale = maximum(abs,A[n+2,:])
    A[idx,:] = scale * I(N)[idx,:]        # Dirichet assignment
    b[idx] = scale * g.(X[idx],Y[idx])    # assigned values

    # Solve the linear sytem and reshape the output.
    u = A\b
    U = reshape(u,m+1,n+1)
    return x,y,U
end

"""
    elliptic(ϕ,g,m,xspan,n,yspan)

Solve the elliptic PDE `ϕ`(x,y,u,u_x,u_xx,u_y,u_yy)=0 on the
rectangle `xspan` x `yspan`, subject to `g`(x,y)=0 on the boundary.
Uses `m`+1 points in x by `n`+1 points in y in a Chebyshev
discretization.

Returns vectors defining the grid and a matrix of grid solution values.
"""
function elliptic(ϕ,g,m,xspan,n,yspan)
    # Discretize the domain.
    x,Dx,Dxx = diffcheb(m,xspan)
    y,Dy,Dyy = diffcheb(n,yspan)
    X = [ x for x in x, y in y ]
    Y = [ y for x in x, y in y ]
    unvec = u -> reshape(u,m+1,n+1)

    # Identify boundary locations and evaluate the boundary condition.
    isboundary = trues(m+1,n+1)
    isboundary[2:m,2:n] .= false
    idx = vec(isboundary)
    gb = g.(X[idx],Y[idx])

    # Evaluate the discretized PDE and its Jacobian, with all the
    # boundary condition modifications applied.
    function residual(u)
        U = unvec(u)
        R = ϕ(X,Y,U,Dx*U,Dxx*U,U*Dy',U*Dyy')
        @. R[idx] = u[idx] - gb
        return vec(R)
    end

    # Solve the equation.
    u = levenberg(residual,vec(zeros(size(X))))[end]
    U = unvec(u)

    return function (ξ,η)
		v = [ chebinterp(x,u,ξ) for u in eachcol(U) ]
		return chebinterp(y,v,η)
    end

end

"Evaluate Chebyshev interpolant with nodes x, values v, at point ξ"
function chebinterp(x,v,ξ)
    n = length(x)-1
    w = (-1.0).^(0:n)
    w[[1,n+1]] .*= 0.5

	terms = @. w / (ξ - x)
	if any(isinf.(terms))     # exactly at a node
		idx = findfirst(ξ.==x)
		return v[idx]
	else
		return sum(v.*terms) / sum(terms)
	end
end


function gauleg(a,b,n)
#==
    1D Gauss-Legendre quadrature rule.

    quadrule = gauleg(a,b,n) computes the n-point Gauss-Legendre
    quadrature rule on the interval [a,b] up to mashine precision.

    Note that all quadrature rules obtained from gauleg are of order 2*n-1.

   The output named tuple quadrule contains the following fields:
    w n-by-1 vector specifying the weights of the quadrature rule.
    X n-by-1 vector specifying the abscissae of the quadrature rule.

   Example:
    quad = gauleg(0,1,10);
    quad.x
    quad.w
=#
    if n < 0
        throw(DomainError(n, "Input n must be a non-negative integer"))
    elseif n==1
        x = 0; w = 2;
    else
        d = zeros(n)
        c = zeros(n-1)
        for i in 1:(n-1)
            c[i]=i/sqrt(4*i*i-1)
        end
        J=LinearAlgebra.SymTridiagonal(d,c)
        x=LinearAlgebra.eigvals(J)
        vec=LinearAlgebra.eigvecs(J)
        w=2*vec[1,:].*vec[1,:]

    end

    # The above rule computed a quadrature rule for the reference interval.
    # The reference interval is always [-1,1], and if [a,b] != [-1,1] then we
    # re-map the abscissas (x) and rescale the weights.

    xm = (b+a)/2; # midpoint
    xl = (b-a)/2; # area of the requested interval / area of the reference interval

    x = xm .+ xl*x;
    w = w * xl;

    return (x=x,w=w) #named tuple
end


using ProgressBars, Plots, StatsBase, LaTeXStrings, JLD, Distributions, StatsPlots

Plots.default(titlefontsize=12, tickfontsize=10, labelfontsize=16,
    fontfamily="Computer Modern")

function UniformRand(;N,a,c,m)
    seed = rand(1:1000)

    series = [seed]
    for n ∈ 1:N+1
        randvalue = (a*series[end] + c) % m
        push!(series,randvalue)
    end
    return series[3:end]
end

# function reshuffle(parameters)
#     return round.(Int,baserand(;parameters...) * (parameters[:N]-1) / parameters[:m] .+ 1)
# end

function FloatUniformRand(N)
    parameters = Dict(:N => N, :a => 214013, :c => 2531011, :m => 2^32)
    return UniformRand(;parameters...) / parameters[:m]
end

function GaussRandDist(σ, μ, N)
    x₁ = FloatUniformRand(round(Int,N/2))
    x₂ = FloatUniformRand(round(Int,N/2))

    ρ = σ * .√(2 * log.(1 ./ (1 .- x₁)))
    θ = 2π * x₂

    y₁ = ρ .* sin.(θ) .+ μ
    y₂ = ρ .* cos.(θ) .+ μ

    return [y₁; y₂]
end

Data = [GaussRandDist(1,3,1000000),
            GaussRandDist(2,6,1000000),
                GaussRandDist(3,9,1000000),
                    GaussRandDist(4,12,1000000),]

cd(dirname(@__FILE__))

save("../../Data/Q2/Q2-samples.jld","Samples", Data)

colors = [:steelblue, :purple, :green, :brown]
histogram(Data[1],  bins = 25,
    label = L"Generated\ Normal:\ \mu =3 , \sigma =1",
    alpha = 0.5, c = colors[1], normalize = true)
plot!(Normal(3,1), c = :black, label = L"Normal Curve:\ \mathcal{N}(\mu =3 , \sigma =1)")

histogram!(Data[2], bins = 50,
    label = L"Generated\ Normal:\ \mu =6 , \sigma =2",
    alpha = 0.5, c = colors[2], normalize = true)
plot!(Normal(6,2), c = :black, label = L"Normal Curve:\ \mathcal{N}(\mu =6 , \sigma =2)")

histogram!(Data[3], bins = 75,
    label = L"Generated\ Normal:\ \mu =9 , \sigma =3",
    alpha = 0.5, c = colors[3], normalize = true)
plot!(Normal(9,3), c = :black, label = L"Normal Curve:\ \mathcal{N}(\mu =9 , \sigma =3)")

histogram!(Data[4], bins = 100,
    label = L"Generated\ Normal:\ \mu =12 , \sigma =4",
    alpha = 0.5, c = colors[4], normalize = true)
plot!(Normal(12,4), c = :black, label = L"Normal Curve:\ \mathcal{N}(\mu =12 , \sigma =4)")
plot!(title = L"Normalized\ Distribution\ of\ Samples\ (10^6\ sample\ for\ each)",frame = :box)

savefig("../../Figs/Q2/Q2-Hist.pdf")


# Michelle Medina
# DATE

using PyPlot

						# 1D TESTS
# ---------------------------------------------------------
p=Params(101,1.0)
dX(p)

#101 points and identity map gives the coordinates
x = functionOnGrid(x->x,Params(100,2));

sinx=functionOnGrid(x->sin(pi*x),Params(100,2));
plot(x,sinx)
show()

					# 1D Convergence Test
H = []
E = []

for N in 50:200
    grid = Params(N,4)
    u = functionOnGrid(x->sin(x),grid)
	#known derivative of sin(x)
    dUdx = functionOnGrid(x->cos(x),grid)

    #L infinity norm: difference between the numerical derivative and
    #the analytical one, while taking the maximum of it's abs. value
    error = maximum(abs.(Deriv1D(u,grid) - dUdx))

    h = dX(grid)
    #append h to the array H and error to the array E
    append!(H,h)
    append!(E,error)

end

#plotting error vs h
loglog(H,E,"r-")

#first order convergence
loglog(H,H,"g--",label="1")

#second order convergence
loglog(H,H.^2,"b--",label="2")
xlabel("h")
ylabel("L Infinity Norm")
legend(loc="best",frameon=false)
show()

# USING L2 NORM
H = []
E = []

for N in 50:200
    grid = Params(N,4)
    u = functionOnGrid(x->sin(x),grid)
    dUdx = functionOnGrid(x->cos(x),grid) #known derivative of sin(x)
    derivative = Deriv1D(u,grid)

    #L 2 norm: sum of all the differences between the numerical derivative and
    #the analytical one squared and then take the square root
    error = sqrt(sum((derivative[2:end-1] - dUdx[2:end-1]).^2))

    h = dX(grid)
    #append h to the array H and error to the array E
    append!(H,h)
    append!(E,error)

end

#plotting error vs h
loglog(H,E,"r-")

#first order convergence
loglog(H,H,"g--",label="1")

#second order convergence
loglog(H,H.^2,"b--",label="2")
xlabel("h")
ylabel("L 2 Norm")
legend(loc="best",frameon=false)
show()

						# 2D TESTS
# ---------------------------------------------------------

sinxcosy = functionOnGrid((x,y)->sin(x)*cos(y), Params2D(100,200,10.0,10.0));

contourf(sinxcosy)
colorbar()

x2y2= functionOnGrid((x,y)->x^2+y^2,Params2D(10,10,1.0,1.0));

contourf(x2y2)
colorbar()

cosxcosy = DerivX_2D(sinxcosy,Params2D(100,100,1.0,1.0));

					# 2D Convergence Test
# Norm for X ; f(x) = sin(pi*x)*cos(pi*y)
Hx = []
Ex = []
Hy = []
Ey = []

for Nx in 50:100, Ny in 50:100
	grid = Params2D(Nx,Ny,4,4)
	u = functionOnGrid((x,y)->sin(pi*x)*cos(pi*y),grid)

	#analytical derivatives
	dUdx = functionOnGrid((x,y)->pi*cos(pi*x)*cos(pi*y),grid)
	dUdy = functionOnGrid((x,y)->sin(pi*x)*(-pi)*sin(pi*y),grid)
	errorx = maximum(abs.(DerivX_2D(u,grid)-dUdx))
	errory = maximum(abs.(DerivY_2D(u,grid)-dUdy))
	hx=dX_2D(grid)
	hy=dY_2D(grid)
	append!(Hx,hx)
	append!(Ex,errorx)
	append!(Hy,hy)
	append!(Ey,errory)
end

loglog(Hx,Ex,"r-")
loglog(Hx,Hx.^2,"--", label="2")
loglog(Hx,Hx,"--",label="1")
xlabel("hx")
ylabel("L Infinity Norm")
legend(loc="best", frameon=false)
show()

# Norm for Y ; f(x) = sin(pi*x)*cos(pi*y)
Hx = []
Ex = []
Hy = []
Ey = []

for Nx in 50:100, Ny in 50:100
    grid = Params2D(Nx,Ny,4,4)
    u = functionOnGrid((x,y)->sin(pi*x)*cos(pi*y),grid)

	#analytical derivatives
    dUdx = functionOnGrid((x,y)->pi*cos(pi*x)*cos(pi*y),grid)
    dUdy = functionOnGrid((x,y)->sin(pi*x)*(-pi)*sin(pi*y),grid)
    errorx = maximum(abs.(DerivX_2D(u,grid)-dUdx))
    errory = maximum(abs.(DerivY_2D(u,grid)-dUdy))
    hx=dX_2D(grid)
    hy=dY_2D(grid)
    append!(Hx,hx)
    append!(Ex,errorx)
    append!(Hy,hy)
    append!(Ey,errory)
end

loglog(Hy,Ey,"r-")
loglog(Hy,Hy.^2,"--", label="2")
loglog(Hy,Hy,"--",label="1")
xlabel("hy")
ylabel("L Infinity Norm")
legend(loc="best", frameon=false)
show()

# Norm for X ; f(x) = x^2+y^2
Hx = []
Ex = []

for Nx in 50:100, Ny in 50:100
    grid = Params2D(Nx,Ny,4,4)
    u = functionOnGrid((x,y)->x^2+y^2,grid)
    #analytical derivatives
    dUdx = functionOnGrid((x,y)->2x,grid)

    errorx = maximum(abs.(DerivX_2D(u,grid)-dUdx))

    hx=dX_2D(grid)

    append!(Hx,hx)
    append!(Ex,errorx)

end

loglog(Hx,Ex,"r-")
loglog(Hx,Hx.^2,"--", label="2")
loglog(Hx,Hx,"--",label="1")
xlabel("hx")
ylabel("L Infinity Norm")
legend(loc="best", frameon=false)
show()

# Norm for Y ; f(x) = x^2+y^2
Hy = []
Ey = []

for Nx in 50:100, Ny in 50:100
    grid = Params2D(Nx,Ny,4,4)
    u = functionOnGrid((x,y)->x^2+y^2,grid)
    #analytical derivatives
    dUdy = functionOnGrid((x,y)->2y,grid)

    errory = maximum(abs.(DerivY_2D(u,grid)-dUdy))

    hy=dY_2D(grid)

    append!(Hy,hy)
    append!(Ey,errory)

end

loglog(Hy,Ey,"r-")
loglog(Hy,Hy.^2,"--", label="2")
loglog(Hy,Hy,"--",label="1")
xlabel("hy")
ylabel("L Infinity Norm")
legend(loc="best", frameon=false)
show()


@doc_str """
    eigenvector_centrality(g)

Compute the eigenvector centrality for the graph `g`.

Eigenvector centrality computes the centrality for a node based on the
centrality of its neighbors. The eigenvector centrality for node `i` is

    ``\mathbf{Ax} = \lambda \mathbf{x}``

where `A` is the adjacency matrix of the graph `g` with eigenvalue `\lambda`.
By virtue of the Perron–Frobenius theorem, there is a unique and positive
solution if `\lambda` is the largest eigenvalue associated with the
eigenvector of the adjacency matrix `A`.

### References

- Phillip Bonacich: Power and Centrality: A Family of Measures.
    American Journal of Sociology 92(5):1170–1182, 1986
    http://www.leonidzhukov.net/hse/2014/socialnetworks/papers/Bonacich-Centrality.pdf
- Mark E. J. Newman: Networks: An Introduction.
       Oxford University Press, USA, 2010, pp. 169.
"""
eigenvector_centrality(g::AbstractGraph) = abs.(vec(eigs(adjacency_matrix(g), nev=1)[2]))::Vector{Float64}


# 04 dimension reduction

# using the libraries
using UMAP, XLSX, VegaDatasets, DataFrames, MultivariateStats, RDatasets, StatsBase, Statistics, LinearAlgebra, Plots, ScikitLearn, MLBase, Distances

# import the data
C = DataFrame(VegaDatasets.dataset("cars"));
describe(C);

# clean the data
dropmissing!(C);
M = Matrix{Float64}(C[!, 2:7]);
n = names(C)[2:end];

# For MLBase and label map
# USA -> 1
# Europe -> 2 etc...
car_origin = C[!, :Origin];
unique(car_origin);
carmap = labelmap(car_origin);
uniqueids = labelencode(carmap, car_origin)

# PCA approximate the matrix
# Center and standardize the data
data = M;
data = (data .- mean(data, dims=1)) ./ std(data, dims=1);

# fit the data
p = fit(PCA, data', maxoutdim=2)

# projection
P = projection(p);
P' * (data[1, :] - mean(p))

#
Yte = P' * data'
Yte = MultivariateStats.transform(p, data')

# reconstruct the matrix
Xr = reconstruct(p, Yte);
data' - Xr;
@show norm(data' - Xr)
# the approximation gets better when more maxoutdims are used.

# plotting
scatter(Yte[1,:],Yte[2,:])

scatter(Yte[1,car_origin.=="USA"],Yte[2,car_origin.=="USA"],color=1,label="USA")
scatter!(Yte[1,car_origin.=="Japan"],Yte[2,car_origin.=="Japan"],color=2,label="Japan")
scatter!(Yte[1,car_origin.=="Europe"],Yte[2,car_origin.=="Europe"],color=3,label="Europe")
xlabel!("pca component1")
ylabel!("pca component2")

# three dim maxxout
p = fit(PCA,data',maxoutdim=3)
Yte = MultivariateStats.transform(p, data')
scatter3d(Yte[1,:],Yte[2,:],Yte[3,:],color=uniqueids,legend=false)

# tsne
# @sk_import manifold : TSNE
tfn = TSNE(n_components=3 ,perplexity=20.0,early_exaggeration=50)
Y2 = tfn.fit_transform(data);
scatter(Y2[:,1],Y2[:,2],Y2[:,3],color=uniqueids,legend=false,size=(400,300),markersize=3)

# umap
L = cor(data,data,dims=2)
@time embedding = umap(L, )

scatter(embedding[1,:],embedding[2,:],color=uniqueids)

# different distance
@time L2 = pairwise(Euclidean(), data, data,dims=1)
embedding = umap(-L, 2)
scatter(embedding[1,:],embedding[2,:],color=uniqueids)


"""
    LUdenganP(A)
adalah fungsi yang digunakan untuk memfaktorisasi matriks `A` sembarang menjadi
dua matriks, yaitu matriks segitiga atas `U` dan bawah `L` serta matriks permutasi P
sedemikian sehingga `LU=PA`.

# Examples
```jl
julia> A = [ 2 5 4 2 3
            4 1 0.4 6 5
            0.9 4 8 7 10
            5 7 0.5 2 7
            2 6 8 9 4];

julia> L,U,P = LUdenganP(A);

julia> L
5×5 Array{Float64,2}:
 1.0    0.0       0.0        0.0       0.0
 0.8    1.0       0.0        0.0       0.0
 0.18  -0.595652  1.0        0.0       0.0
 0.4   -0.695652  0.986094   1.0       0.0
 0.4   -0.478261  0.480405  -0.537685  1.0

julia> U
5×5 Array{Float64,2}:
  5.0           7.0  0.5   2.0       7.0
  0.0          -4.6  0.0   4.4      -0.6
  1.11022e-16   0.0  7.91  9.26087   8.38261
 -1.09478e-16   0.0  0.0   2.12879  -7.48343
 -1.122e-16     0.0  0.0   0.0      -8.13773

julia> P
5×5 Array{Bool,2}:
 0  0  0  1  0
 0  1  0  0  0
 0  0  1  0  0
 0  0  0  0  1
 1  0  0  0  0
```
return matriks `L`, `U`, dan `P`.
"""
function LUdenganP(A)
    # Definisikan matriks L dan P.
    n,n = size(A);
    L = zeros(n,n);
    P = Array(I(n));
    # Lakukan operasi baris dasar terhadap matriks gandeng
    Aug = copy(A)
    for p = 1:n-1
        # Lakukan operasi pindah baris untuk menentukan nilai pivot
        val,j = findmax(abs.(Aug[p:n,p]));
        # Pivoting U
        C = Aug[p,:];
        Aug[p,:] = Aug[j+p-1,:];
        Aug[j+p-1,:] = C;
        # Pivoting L
        C = L[p,:];
        L[p,:] = L[j+p-1,:];
        L[j+p-1,:] = C;
        # Pivoting P
        C = P[p,:];
        P[p,:] = P[j+p-1,:];
        P[j+p-1,:] = C;
        # Jika pivot bernilai nol, maka gagal.
        if Aug[p,p]==0
            error("Matriks Singular")
        end
        # Lakukan eliminasi dan simpan pengali.
        for i = p+1:n
            k = Aug[i,p]/Aug[p,p]
            Aug[i,1:n] = Aug[i,1:n] - k*Aug[p,1:n];
            L[i,p] = k;
        end
    end
    U = Aug;
    L = L + I(n);
    return L,U,P
end


function gausslobatto( n )
# Gauss-Legendre-Lobatto Quadrature Nodes and Weights

    if ( n == 1 )
        error("Lobatto undefined for n = 1.")
    elseif ( n == 2 )
        x = ([-1.0,1.0],[1.0,1.0])
    elseif ( n == 3 )
        x = ([-1.0, 0.0, 1.0], [1.0, 4.0, 1.0]/3)
    else
        # Compute via GaussJacobi:
        x = gaussjacobi(n - 2, 1.0, 1.0)
        w = x[2]; x = x[1];
        x = [-1.0, x, 1.0]
        w = [-1.0, w, 1.0]
        w = w./(1-x.^2)
        w[1] = 2/(n*(n - 1))
        w[n] = 2/(n*(n - 1))
        x = (x,w)
    end
    return x
end


using LinearAlgebra
using SparseArrays
using Plots
using ForwardDiff

uexact(x) = log(2+sin(pi*x))
dudx_exact(x) = ForwardDiff.derivative(uexact,x)
f(x) = -ForwardDiff.derivative(dudx_exact,x) # -d2u/d2x = f
α = uexact(-1)
β = uexact(1) - dudx_exact(1)

function solve(m,f,α,β)
    x = LinRange(-1,1,m+2) # add 2 endpoints
    xint = x[2:end]
    h = x[2]-x[1]
    A = (1/h^2) * spdiagm(0=>2*ones(m+1),-1=>-ones(m),1=>-ones(m))
    A[end,end] = (1-h)/h^2
    b = f.(xint)
    b[1] += α/h^2
    b[end] = .5*f(x[end]) - β/h
    u = A\b
    return u,x,xint,h
end

mvec = 2 .^ (2:9)
hvec = zeros(length(mvec))
err_inf = zeros(length(mvec))
for (i,m) in enumerate(mvec)
    u,x,xint,h = solve(m,f,α,β)
    hvec[i] = h
    err_inf[i] = maximum(@. abs(uexact(xint) - u))
end

plot(hvec,err_inf,marker=:dot,label="Max error")
plot!(hvec,hvec.^2,linestyle=:dash,label="O(h^2)")
plot!(xaxis=:log,yaxis=:log,legend=:bottomright)


function I(n)
    mat = zeros(n, n)
    for i in 1:n
        mat[i, i] = 1.0
    end
    return mat
end

mean(x) = sum(x)/length(x)

function var(x)
    m = mean(x)
    xm = x .- m
    return sum((xm .* xm) / (length(x) - 1))
end

std(x) = var(x) |> sqrt

geomean(x::Array{<: Number, 1}) = exp(sum(log.(x))/length(x))

function cor(x, y)
    xmd= x .- mean(x)
    ymd =y .- mean(y)
    return (sum(xmd .* ymd) / sqrt(var(x) * var(y))) / (length(x) - 1.0)
end

function cor(x::Matrix{Float64})
    _, p = size(x)
    cormat = ones(p, p)
    for i in 1:p
        for j in i:p
            if i != j
                cr = cor(x[:,i], x[:,j])
                cormat[i, j] = cr
                cormat[j, i] = cr
            end
        end
    end
    return cormat
end

function euclidean(v1::Array{T1,1}, v2::Array{T2,1})::Float64 where {T1 <: Number,T2 <: Number}
    (v1 .- v2).^2.0 |> sum |> sqrt
end

function euclidean(v1::Array{T,1})::Float64 where T <: Number
    v2 = zeros(Float64, length(v1))
    return euclidean(v1, v2)
end

function normalize(v1::Array{T,1})::Array{Float64,1} where T <: Number
    return v1 ./ euclidean(v1)
end

function normalize(data::DataFrame)::DataFrame
    df = similar(data)
    _, p = size(df)
    for i in 1:p
        df[:, i] = normalize(data[:, i])
    end
    return df
end

function apply_columns(fs::Array{Function,1}, data::Union{DataFrame,Array{T,2}} where T <: Number)
    _, m = size(data)
    return [fs[i](data[:,i]) for i in 1:m]
end

function apply_columns(f::Function, data::Union{DataFrame,Array{T,2}} where T <: Number)
    return [f(c) for c in eachcol(data)]
end

function colmins(data::DataFrame)
    return apply_columns(minimum, data)
end

function colmaxs(data::DataFrame)
    return apply_columns(maximum, data)
end

function colsums(data)::Array{Number,1}
    return apply_columns(sum, data)
end

function colmeans(data)::Array{Number,1}
    return apply_columns(mean, data)
end

function apply_rows(f::Function, data::Union{DataFrame,Array{T,2}} where T <: Number)
    return [f(c) for c in eachrow(data)]
end

function rowmins(data::DataFrame)::Array{Float64,1}
    return apply_rows(minimum, data)
end

function rowmaxs(data::DataFrame)::Array{Float64,1}
    return apply_rows(maximum, data)
end

function rowsums(data)::Array{Float64,1}
    return apply_rows(sum, data)
end

function rowmeans(data)::Array{Float64,1}
    return apply_rows(mean, data)
end

function unitize(v::Array{T,1})::Array{Float64,1} where {T <: Number}
    return v ./ sum(v)
end

function Base.:*(w::Array{Float64,1}, data::DataFrame)::DataFrame
    newdf = copy(data)
    _, p = size(newdf)
    for i in 1:p
        newdf[!, i] = convert.(Float64, newdf[!,i])
        newdf[:, i] = w[i] .* data[:, i]
    end
    return newdf
end

function Base.:-(r1::DataFrameRow, r2::DataFrameRow)::Array{Number,1}
    # v1 = convert(Array{Float64,1}, r1)
    v1 = Vector{Float64}(r1)
    # v2 = convert(Array{Float64,1}, r2)
    v2 = Vector{Float64}(r2)
    return v1 .- v2
end

function Base.:-(r1::Array{T,1}, r2::DataFrameRow)::Array{T,1} where T <: Number
    # v2 = convert(Array{Float64,1}, r2)
    v2 = Vector{Float64}(r2)
    return r1 .- v2
end

function Base.:-(r1::DataFrameRow, r2::Array{T,1})::Array{T,1} where T <: Number
    v1 = convert(Array{Float64,1}, r1)
    return v1 .- r2
end

"""
    makeDecisionMatrix(mat; names)

    Create a DataFrame using a decision matrix optionally using column names.

# Arguments:
 - `mat::Array{T,2}`: Matrix of numbers in any set
 - `names::Union{Nothing,Array{String,1}}`: Column names. Default is nothing. If column names are not given, they are labelled as Crt 1, Crt 2, ..., etc.

"""
function makeDecisionMatrix(mat::Array{T,2}; names::Union{Nothing,Array{String,1}}=nothing)::DataFrame where {T <: Number}
    _, m = size(mat)
    df = DataFrame()
    for i in 1:m
        if names isa Nothing
            name = string("Crt", i)
        else
            name = names[i]
        end
        df[:,Symbol(name)] = convert(Array{Float64,1}, mat[:, i])
    end
    return df
end

function Base.minimum(df::DataFrame)
    df |> Matrix |> minimum
end

function reverseminmax(fns::Array{Function,1})::Array{Function,1}
    newfs = map(x -> if x == minimum maximum else minimum end, fns)
    return newfs
end

function makeminmax(fns::Array{K,1} where K)::Array{Function,1}
    return convert(Array{Function,1}, fns)
end


# Practical Review: Numeric Optimization (Ch. 12)

##
# Let's start with a simple problem, where we already know the
# answers. We will apply numeric optimization methods and we
# will be able to verify that they give use the correct results.
# To start, let's define some data, and do the basic OLS
# computations
using Econometrics
n = 5
x = [ones(n) 1:n]
β = [10., -1.]
ϵ = randn(n)
y = x*β + ϵ
ols(y,x)

##
# Let's see how to get these results using numeric minimization
# and extremum estimation theory...
using Econometrics, Statistics
objᵢ = β -> (y-x*β).^2.  	# iᵗʰ obs contrib to sum of squares
obj = β -> mean(objᵢ(β))	# average objective
fminunc(obj, zeros(2))

##
# fminunc is just a frontend to the Optim.jl package's LBFGS unconstrained
# minimizer routine. To call that directly, do
using Optim
tol = 1e-08
results = Optim.optimize(obj, zeros(2), LBFGS(),
                            Optim.Options(
                            g_tol = tol,
                            x_tol=tol,
                            f_tol=tol); autodiff=:forward)

##
# The output has the function value and convergence information. To see the
# value of the minimizer, do
βhat = results.minimizer

##
# Let's check that the Hessian is p.d.
using ForwardDiff
H = ForwardDiff.hessian(obj, βhat)
##
using LinearAlgebra
eigen(H)

##
# The eigenvalues are all positive, so the Hessian is p.d. at
# the solution. For the OLS case, the objective function is
# known to be globally convex, so multiple local optima are not
# a problem. Let's look at another problem, which has multiple
# minima, and see how to find the global minimizer
# Here's another objective function
f(x,y) = -(3*(1-x)^2 * exp(-(x^2) - (y+1)^2)
    - 6*(x/5 - x^3 - y^5) * exp(-x^2 - y^2)
    - 1/3 * exp(-(x+1)^2 - y^2))

##
# let's explore it:
using Plots
plotly() # this backend makes nice 3d plots
x = range(-4, step=0.1, stop=4)
y = x
surface(x, y, (x,y)->f(x,y),c=:viridis)
xlabel!("x")
ylabel!("y")
##
contour(x,y,(x,y)->f(x,y),c=:viridis)

##
# This function has 3 local min, and 2 local max.
# The global minimizer is near (0,1.6), with a function
# value of -4.86. We will need some care to find the global
# minimizer. The domain is (-4,4)×(-4,4), which may help us.
# First, let's try gradient-based minimization, starting at
# a random point in the domain.

##
# Before we do that, the form of the function defined above
# does not take a single vector argument. For optimization, we
# need all parameters in a single vector. Julia's multiple
# dispatch allow us to do this easily, by defining another
# method for the function f:
function f(θ)
   x,y = θ
   f(x,y) # pretty neat, eh?
end

##
#=
Another thing to check before doing optimization! Remember
that gradient based methods will work better when the data
are scaled to put the elements of the gradient one a common
order of magnitude. You can check this by examining the
gradient at some points. Run this block a few times to check.
=#
using ForwardDiff
lower = [-4., -4.]
upper = -lower
θstart = 8.0 .* rand(2) .-4.0
ForwardDiff.gradient(f, θstart)
# in this case, things seem to be ok.

##
# Now we can try optimization. Running this block repeatedly (Alt-enter)
# will show that each of the 3 minimizers are found. Thus, if one defined
# LBFGS with enough start values, the global minimizer would be found.
# If you run the code enough times, you will see that the algorithm
# occasionally goes off far away from the 3 local minimizers. In that
# region, the gradients are zero, as we see in the plots, and the algorithm
# can't find better points.
using Econometrics, Plots, Optim
gr() # using the GR backend
x = range(-4, step=0.1, stop=4)
y = x
contour(x,y,f,c=:viridis)
θstart = 8.0 .* rand(2) .-4.0
tol = 1e-08
results = Optim.optimize(f, θstart, LBFGS(),
                            Optim.Options(
                            g_tol = tol,
                            x_tol=tol,
                            f_tol=tol); autodiff=:forward)

θhat = results.minimizer
scatter!([θhat[1]],[θhat[2]],legend=false, markersize=10, c = :viridis)

##
# A more robust algorithm is to use the known domain to define
# box constraints. This will still bounce between the 3 local
# min, but it will not shoot off to far away points.
using Econometrics, Plots, Optim
gr() # using the GR backend
x = range(-4, step=0.1, stop=4)
y = x
contour(x,y,f, c=:viridis)
lower = [-4., -4.]
upper = -lower
θstart = 8.0 .* rand(2) .-4.0
tol = 1e-08
results = Optim.optimize(f, lower, upper, θstart, Fminbox(LBFGS()),
                            Optim.Options(
                            g_tol = tol,
                            x_tol=tol,
                            f_tol=tol); autodiff=:forward)
θhat = results.minimizer
scatter!([θhat[1]],[θhat[2]],legend=false, markersize=10)

##
#
# Global optimization
#
# Another option is to use a global optimizer. An example is the
# simulated annealing algorithm. This one should find the global
# min every time, provided the cooling schedule is slow enough
using Econometrics
# try setting rt=0.05, it will fail sometimes, because the
# search region is shrunk too quickly
# rt=0.9 has worked every time I've tried it
rt = 0.75
lower = [-4., -4.]
upper = -lower
θstart = 8.0 .* rand(2) .-4.0
samin(f, θstart, lower, upper, rt=rt);
println("for reference, the global minimizer is (-0.00879, 1.58163)")
println("with the function value -4.8611920055")

#=
To summarize, a gradient based minimizer should be reliable
when the objective function is globally convex. When this is
not the case, one can use multiple start values for a gradient
based method, or one might prefer to use a global minimizer.
=#


using LinearAlgebra

#Givens from Problem Statement
#Shear Modulus(psi),diameter(in),length(in),torque(lb*in)
G = 3.9*10^6
G2 = 4.0*10^6
d = 1.5
d2 = 1
l = 2.5*12
l2 = 1*12
l3 =2*12
τ = -200*12

# Polar Second Area Moment of a Cylinder (in^4)
J_cyl(d) = 0.5*pi*(d/2)^4
J = J_cyl(d)
J2 = J_cyl(d2)

# Torsional Stiffness Function
k(G,J,l) = (G*J)/l

# k_local(Elastic Modulus,thickness,height,length,number of nodes,
# row/column one,row/column two, ==1 left fixed/ ==2 right fixed)
function k_local(G,J,l,i,n1,n2,fixed=0)
    k_global = zeros(i,i)
    kin = k(G,J,l)
    k_global[n1,n1] = kin
    k_global[n1,n2] = -kin
    k_global[n2,n1] = -kin
    k_global[n2,n2] = kin
    if fixed == 1
      adj = zeros(1,i)
      adj[n1] = 1
      k_global[n1,:] = adj
      k_global[:,n1] = adj
    elseif fixed == 2
      adj = zeros(1,i)
      adj[n2] = 1
      k_global[n2,:] = adj
      k_global[:,n2] = adj
    end
    return k_global
end

# Global for angular displacement calculation
k_1 = k_local(G,J,l,4,1,2,1)
k_2 = k_local(G,J,l2,4,2,3)
k_3 = k_local(G2,J2,l3,4,3,4,2)
kG = k_1+k_2+k_3

#Global for Reaction calculation
k_11 = k_local(G,J,l,4,1,2)
k_22 = k_local(G,J,l2,4,2,3)
k_33 = k_local(G2,J2,l3,4,3,4)
kGG = k_11+k_22+k_33

# Torque Vector
t = Float64[0 τ 0 0]
T = transpose(t)

# Angular Displacement Vector
θ = kG\T
println("Nodal Angular Displacement Vector (radians)")
display(θ)

# Moment Reaction Vector
τ_app = kGG*θ
println("Moment Reaction Vector (lb*in):")
display(τ_app)


function collatz_chain_length(x::Int)
	if x==1
		return 1
	end
	if x%2==0
		x=Int(x/2)
	else
		x=x*3+1
	end
	return collatz_chain_length(x)+1
end

function collatz_length(x::Int)
	if x==1
		return 1
	end
	count=1;
	while x!=1
		count+=1
		if x%2==0
			x=Int(x/2)
			if x==1
				break
			end
		else
			x=x*3+1
		end

	end
	return count
end

function compute()
    const N=1000000
    ans =0
    index=1
	for i=1:N
		if ans<collatz_chain_length(i)
			index=i
			ans=collatz_chain_length(i)
		end
	end

	return index
end

function compute2()
	 const N=1000000
    ans =0
    index=1
	for i=1:N
		if ans<collatz_length(i)
			index=i
			ans=collatz_length(i)
		end
	end

	return index
end

print(compute2())


##
using Roots
function GE(π, γ, ω1A, ω1B, ω2A, ω2B)
    f(p) = π * (ω1A + p*ω1B) + ((ω2A + p*ω2B) / (1 + p * ((1 - π)/(p * π))^(1/γ))) - ω1A - ω2A
    p = round(find_zero(f, 1.5), digits= 3)
    C1A = π * (ω1A + p*ω1B)
    C1B = ((1 - π) / p) * (ω1A + p*ω1B)
    C2A = (ω2A + p*ω2B) / (1 + p * ((1 - π)/(p * π))^(1/γ))
    C2B = ((ω2A + p*ω2B) / (1 + p * ((1 - π)/(p * π))^(1/γ))) * ((1 - π)/(p * π))^(1/γ)
    print("p = $p \n", "C1A = $C1A \n", "C1B = $C1B \n", "C2A = $C2A \n", "C2B = $C2B \n")
end
##
#4(e) No aggregate uncertanity
GE(0.5, 2, 2, 5, 5, 2)
#4(f) There is no aggregate uncertainty but for person A the good world is more likely to happen
GE(0.25, 2, 2, 5, 5, 2)
#4(f) There is no aggregate uncertainty but for person B the good world is more likely to happen
GE(0.75, 2, 2, 5, 5, 2)
#4(g)(e) γ=1.5
GE(0.5, 1.5, 2, 5, 5, 2)
#4(g)(e) γ=3
GE(0.5, 3, 2, 5, 5, 2)
#4(g)(f) γ=1.5 π=0.25
GE(0.25, 1.5, 2, 5, 5, 2)
#4(g)(f) γ=1.5 π=0.75
GE(0.75, 1.5, 2, 5, 5, 2)
#4(g)(f) γ=3 π=0.25
GE(0.25, 3, 2, 5, 5, 2)
#4(g)(f) γ=3 π=0.75
GE(0.75, 3, 2, 5, 5, 2)
#4(g)(e) γ=2 π=0.5
GE(0.5, 2, 2, 3, 5, 2)
#4(g)(e) γ=1.5 π=0.5
GE(0.5, 1.5, 2, 3, 5, 2)
#4(g)(e) γ=3 π=0.5
GE(0.5, 3, 2, 3, 5, 2)


export logsoftmax, logsumexp, softmax, logp
using AutoGrad: AutoGrad, @primitive1, value

"""
    softmax(x; dims=:)
    logsoftmax(x; dims=:)

Treat entries in `x` as as unnormalized log probabilities and return normalized (log)
probabilities, i.e.

    softmax(x; dims) = exp.(x) ./ sum(exp.(x); dims=dims)
    logsoftmax(x; dims) = x .- log.(sum(exp.(x); dims=dims))

For numerical stability `x = x .- maximum(x,dims=dims)` is performed before exponentiation.

`dims` is an optional argument, if not specified the normalization is over the whole `x`,
otherwise the normalization is performed over the given dimensions.  In particular, if `x` is
a matrix, `dims=1` normalizes columns of `x` and `dims=2` normalizes rows of `x`.

"""
softmax, logsoftmax, logp

"""
    logsumexp(x;dims=:)

Compute `log(sum(exp(x);dims))` in a numerically stable manner.

`dims` is an optional argument, if not specified the summation is over the whole `x`,
otherwise the summation is performed over the given dimensions.  In particular if `x` is a
matrix, `dims=1` sums columns of `x` and `dims=2` sums rows of `x`.

"""
logsumexp

# Separate user facing methods from underlying implementations
logsoftmax(x; dims=:) = _logsoftmax(x, dims=dims)
softmax(x; dims=:)    = _softmax(x, dims=dims)
logsumexp(x; dims=:)  = _logsumexp(x, dims=dims)
const logp = logsoftmax

function _logsoftmax(x; dims)
    isa(x, Number) && return zero(x)
    isempty(x) && return similar(x)
    x = x .- maximum(x,dims=dims)
    return (x .- log.(sum(exp.(x); dims=dims)))
end

function ∇logsoftmax(x,y,dy; dims)
    return (dy - exp.(y).*sum(dy; dims=dims))
end

function _softmax(x; dims)
    isa(x, Number) && return one(x)
    isempty(x) && return similar(x)
    x = x .- maximum(x; dims=dims)
    x = exp.(x)
    return (x ./ sum(x; dims=dims))
end

function ∇softmax(x,y,dy; dims)
    return (y .* dy .- y .* sum(y .* dy; dims=dims))
end

function _logsumexp(x; dims)
    isa(x, Number) && return x
    isempty(x) && throw(ArgumentError("reducing over an empty collection is not allowed"))
    xmax = maximum(x,dims=dims)
    return (xmax + log.(sum(exp.(x .- xmax),dims=dims)))
end

function ∇logsumexp(x,y,dy; dims)
    return (dy .* exp.(x .- y))
end

@primitive1 _logsoftmax(x;dims),dy,y  ∇logsoftmax(x,y,dy,dims=dims)
@primitive1 _softmax(x;dims),dy,y     ∇softmax(x,y,dy,dims=dims)
@primitive1 _logsumexp(x;dims),dy,y   ∇logsumexp(x,y,dy;dims=dims)

# Math for the softmax loss: x is unnormalized input, p is target probabilities, q is
# estimated probabilities. Read left column down, right column (loss gradients) back up.
#
# x			dx = -p + qz/z = -p + exp(logq)
# xmax  = max(x,1)	-sum(db)=0
# logqz = x .- xmax	-p + qz/z
# qz    = exp(logqz)	rep(1/z)
# z     = sum(qz,1)	1/z
# logz  = log(z)	sum(p)=1
# logq  = logqz.-logz	-p
# plogq = p .* logq	-1
# loss  = -sum(plogq)	1


#=
                            Largest prime factor

    The prime factors of 13195 are 5, 7, 13 and 29.

    What is the largest prime factor of the number 600851475143 ?

                            Answer: 6857
=#

include("../project_euler.jl")
using project_euler

print(pop!(prime_factors(600851475143)), "\n")


module Loss

export AbstractLoss, ALoss, Float, loss, ∇loss, SSE, SoftmaxXEntropy, softmax

include("./tensor_dt.jl")
using .TensorDT: Tensor

## ======================================================================
## Loss
## ======================================================================
abstract type AbstractLoss end
const ALoss = AbstractLoss
const Float = Float64

loss(::AbstractLoss, _ŷ::Tensor, _y::Tensor) = throws(ArgumentError("Not Implemented"))
∇loss(::AbstractLoss, _ŷ::Tensor, _y::Tensor) = throws(ArgumentError("Not Implemented"))

#### Sum squared Error (SSE)

struct SSE <: AbstractLoss
end

function loss(_self::SSE, ŷ::Tensor, y::Tensor)::Float
  sum((ŷ .- y) .^ 2)
end

function ∇loss(_self::SSE, ŷ::Tensor, y::Tensor)::Tensor
  2 * (ŷ .- y)
end

struct SoftmaxXEntropy <: AbstractLoss
end

function loss(_self::SoftmaxXEntropy, ŷ::Tensor, y::Tensor; ϵ=1e-20)::Float
  -sum(log.(softmax(ŷ) .+ ϵ) .* y)
end

function ∇loss(_self::SoftmaxXEntropy, ŷ::Tensor, y::Tensor)::Tensor
    softmax(ŷ) .- y
end

## Softmax function helper

function softmax(tensor::Tensor)::Tensor
  """Softmax along ths last dimension"""
  ## for numerical stability - subtract largest value
  exps = exp.(tensor .- maximum(tensor))
  sum_exps = sum(exps)
  exps ./ sum_exps
end

end  ## Module


using CSV, Distributions, HypothesisTests

data1 = CSV.read("../data/machine1.csv", header=false)[:,1]
data2 = CSV.read("../data/machine2.csv", header=false)[:,1]
xBar1, xBar2 = mean(data1), mean(data2)
s1, s2 = std(data1), std(data2)
n1, n2 = length(data1), length(data2)
alpha = 0.05

v = (s1^2/n1 + s2^2/n2)^2 / ( (s1^2/n1)^2 / (n1-1) + (s2^2/n2)^2 / (n2-1) )

t = quantile(TDist(v),1-alpha/2)

println("Calculating formula: ", (xBar1 - xBar2 - t*sqrt(s1^2/n1 + s2^2/n2),
				  xBar1 - xBar2 + t*sqrt(s1^2/n1 + s2^2/n2)))
println("Using confint(): ",   confint(UnequalVarianceTTest(data1,data2),alpha))


# Consider quadratic Diophantine equations of the form:
#
# x^2 - D * y^2 = 1
#
# For example, when D=13, the minimal solution in x is 649^2 - 13 × 180^2 = 1.
#
# It can be assumed that there are no solutions in positive integers when D is
# square.
#
# By finding minimal solutions in x for D = {2, 3, 5, 6, 7}, we obtain the
# following:
#
# 3^2 - 2×2^2 = 1
# 2^2 - 3×1^2 = 1
# 9^2 - 5×4^2 = 1
# 5^2 - 6×2^2 = 1
# 8^2 - 7×3^2 = 1
#
# Hence, by considering minimal solutions in x for D ≤ 7, the largest x is
# obtained when D=5.
#
# Find the value of D ≤ 1000 in minimal solutions of x for which the largest
# value of x is obtained.

using ProjectEulerSolutions

# This format for a Diophantine equation is called Pell's Equation.  It turns
# out that the minimal solution for x given a particular D is identified by
# finding the continued fraction convergent that approximates sqrt(D), and
# substituting into Pell's equation until it equals 1.  It turns out that the
# minimal values of x can get very large, like for D=61
# (https://oeis.org/A002350).  Since a quick solution for the minimal value is
# x is found, just cycle through D values storing the D with the largest x.
function p066solution(n_max::Integer=3)::Integer

    x_max = 0
    d_max = 0
    for D in 2:n_max
        if sqrt(D) == floor(sqrt(D))  # No solution for D = perfect square
            continue
        end

        # Need to use BigInts since the numerators and denominators can get
        # quite large
        a0 = BigInt(floor(sqrt(D)))
        a = a0
        m = 0
        d = 1
        A = Array{BigInt,1}([1, a0])
        B = Array{BigInt,1}([0, 1])
        k = 1
        while A[end]*A[end] - D*B[end]*B[end] != 1
            m = d * a - m
            d = div(D - m * m, d)
            a = div(a0 + m, d)
            append!(A, a * A[k+1] + A[k])
            append!(B, a * B[k+1] + B[k])
            k += 1
        end
        if x_max < A[end]
            x_max = A[end]
            d_max = D
        end
    end
    return d_max
end

p066 = Problems.Problem(p066solution)

Problems.benchmark(p066, 1000)


"""
    SqExponentialKernel()

The squared exponential kernel is a Mercer kernel given by the formula:
```
    κ(x, y) = exp(-‖x - y‖² / 2)
```
Can also be called via `RBFKernel`, `GaussianKernel` or `SEKernel`.
See also [`ExponentialKernel`](@ref) for a
related form of the kernel or [`GammaExponentialKernel`](@ref) for a generalization.
"""
struct SqExponentialKernel <: SimpleKernel end

kappa(κ::SqExponentialKernel, d²::Real) = exp(-d² / 2)

metric(::SqExponentialKernel) = SqEuclidean()

iskroncompatible(::SqExponentialKernel) = true

Base.show(io::IO,::SqExponentialKernel) = print(io,"Squared Exponential Kernel")

## Aliases ##
const RBFKernel = SqExponentialKernel
const GaussianKernel = SqExponentialKernel
const SEKernel = SqExponentialKernel

"""
    ExponentialKernel()

The exponential kernel is a Mercer kernel given by the formula:
```
    κ(x,y) = exp(-‖x-y‖)
```
"""
struct ExponentialKernel <: SimpleKernel end

kappa(κ::ExponentialKernel, d::Real) = exp(-d)

metric(::ExponentialKernel) = Euclidean()

iskroncompatible(::ExponentialKernel) = true

Base.show(io::IO, ::ExponentialKernel) = print(io, "Exponential Kernel")

## Alias ##
const LaplacianKernel = ExponentialKernel

"""
    GammaExponentialKernel(; γ = 2.0)

The γ-exponential kernel [1] is an isotropic Mercer kernel given by the formula:
```
    κ(x,y) = exp(-‖x-y‖^γ)
```
Where `γ > 0`, (the keyword `γ` can be replaced by `gamma`)
For `γ = 2`, see `SqExponentialKernel` and `γ = 1`, see `ExponentialKernel`.

[1] - Gaussian Processes for Machine Learning, Carl Edward Rasmussen and Christopher K. I.
    Williams, MIT Press, 2006.
"""
struct GammaExponentialKernel{Tγ<:Real} <: SimpleKernel
    γ::Vector{Tγ}
    function GammaExponentialKernel(; gamma::T=2.0, γ::T=gamma) where {T<:Real}
        @check_args(GammaExponentialKernel, γ, γ >= zero(T), "γ > 0")
        return new{T}([γ])
    end
end

@functor GammaExponentialKernel

kappa(κ::GammaExponentialKernel, d::Real) = exp(-d^first(κ.γ))

metric(::GammaExponentialKernel) = Euclidean()

iskroncompatible(::GammaExponentialKernel) = true

function Base.show(io::IO, κ::GammaExponentialKernel)
    print(io, "Gamma Exponential Kernel (γ = ", first(κ.γ), ")")
end


#!/usr/bin/julia

function base10ToAny(in_dig,base)
    if base >= 21
        println("Warning: There're not special characters for digits in base 21 or more")
    end
    digits_b10=[]
    q=in_dig
    while true
        if q % base <= 9
            push!(digits_b10,q % base)
        elseif q % base == 10
            push!(digits_b10,'A')
        elseif q % base == 11
            push!(digits_b10,'B')
        elseif q % base == 12
            push!(digits_b10,'C')
        elseif q % base == 13
            push!(digits_b10,'D')
        elseif q % base == 14
            push!(digits_b10,'E')
        elseif q % base == 15
            push!(digits_b10,'F')
        elseif q % base == 16
            push!(digits_b10,'G')
        elseif q % base == 17
            push!(digits_b10,'H')
        elseif q % base == 18
            push!(digits_b10,'I')
        elseif q % base == 19
            push!(digits_b10,'J')
        else
            r=(q % base)
            push!(digits_b10,"($r)")
        end
        q=div(q,base)
        if q == 0
            break
        end
    end
    l=length(digits_b10)
    for idx in 1:l
        print(pop!(digits_b10))
    end
end

function anyBaseTo10(iterDig,base)
    if base >= 21
        println("Error: Only works for a base 20 or less")
        return
    end
    if !isa(iterDig,AbstractString)
        println("Error: Input n should be a string")
        return
    end
    summation=0
    l=sizeof(iterDig)
    for idx in 1:l
        digit = iterDig[ l - (idx-1)]
        if digit == 'A'
            digit = 10
        elseif digit == 'B'
            digit = 11
        elseif digit == 'C'
            digit = 12
        elseif digit == 'D'
            digit = 13
        elseif digit == 'E'
            digit = 14
        elseif digit == 'F'
            digit = 15
        elseif digit == 'G'
            digit = 16
        elseif digit == 'H'
            digit = 17
        elseif digit == 'I'
            digit = 18
        elseif digit == 'J'
            digit = 19
        else
            digit = parse(Int64, digit)
        end
        summation = digit*(base^(idx-1)) + summation
    end
    println(summation)
end

function binAddition(in_bin_1,in_bin_2)
    ib1=string(in_bin_1)
    ib2=string(in_bin_2)
    l1=sizeof(ib1)
    l2=sizeof(ib2)
    if l1 > l2
        AddZeros = l1 - l2
        ib2=("0" ^ AddZeros) * ib2
        larger=l1
    elseif l2 > l1
        AddZeros = l2 - l1
        ib1=("0" ^ AddZeros) * ib1
        larger=l2
    else
        larger=l1
    end
    c=0
    all_s=[]
    for idx in 1:larger
        dig1= parse(Int64, ib1[ larger - (idx-1) ])
        dig2= parse(Int64, ib2[ larger - (idx-1) ])
        d=div( dig1 + dig2 + c, 2)
        s=dig1 + dig2 + c - 2d
        c=d
        push!(all_s,s)
    end
    push!(all_s,c)
    justZero = true
    #println("all_s is:", all_s)
    for idx in 1:length(all_s)
        if length(all_s) == 2 && join(all_s) == "00"
            print(all_s[ length(all_s) - (idx -1 ) ])
            break
        end
        if justZero && all_s[ length(all_s) - (idx -1 ) ] == 0
            continue
        end
        if all_s[ length(all_s) - (idx -1 ) ]  == 1
            justZero = false
        end
        print(all_s[ length(all_s) - (idx -1 ) ])
    end
end

function binMultiplication(a,b)
end

#21 * 21
#441
#110111001

a=10101
b=10101
a=string(a)
b=string(b)
la=sizeof(a)
lb=sizeof(b)
if la > lb
    AddZeros = la - lb
    b=("0" ^ AddZeros) * b
    larger=la
elseif lb > la
    AddZeros = lb - la
    a=("0" ^ AddZeros) * a
    larger=lb
else
    larger=la
end
c_partials=[]
for idx in 1:larger
    #println("b[larger - (idx -1)] is\t", b[larger - (idx -1)])
    if b[larger - (idx -1)] == '1'
        c = a * ("0" ^ (idx - 1))
        #println("This is a: \t",a)
        push!(c_partials,c)
    else
        c = "0"
        push!(c_partials,c)
    end
end
p = 0
#println("This is c_partials: \t",c_partials)
for partial in c_partials
    println("Partial is $partial")
    partial = parse(Int64, partial)

    println("Value of function is: ", binAddition(p,partial) )
    p = binAddition(p,partial)
end


function chebyshev_gauss_lobatto(n::S,domain = [1.0,-1.0]) where {S<:Integer}

  # These nodes a just the Chebyshev extrema by a different name.

  # Construct the nodes on the [-1.0,1.0] interval

  if n <= 0
    error("The number of nodes must be positive.")
  end

  if n == 1
    nodes   = [0.0]
    weights = [1.0*pi]
  else
    nodes    = zeros(n)
    nodes[1] = -1.0
    nodes[n] = 1.0

    weights    = zeros(n)
    weights[1] = pi/(2*(n-1))
    weights[n] = pi/(2*(n-1))

    for i = 2:div(n,2)
      x = cos(pi*(i-1)/(n-1))
      nodes[i]       = -x
      nodes[end-i+1] = x
      weights[i]       = pi/(n-1)
      weights[end-i+1] = pi/(n-1)
    end

    if isodd(n)
      nodes[round(Int,(n+1)/2)]   = 0.0
      weights[round(Int,(n+1)/2)] = pi/(n-1)
    end
  end

  # Scale the nodes to the desired domain

  nodes = scale_nodes(nodes,domain)

  return nodes, weights

end


#=
A function f is Lipschitz continous on [a, b]
    |f(x)-f(y)| < l*|x-y|    for all x, y \in [a, b], l > 0
l is as large as the largest unsigned instantaneous
rate of change the function attains on [a,b].

Given a point (x0, f(x0)), The line f(x0)-l(x-x0) for x > x0
and f(x0) + l(x-x0) for x < x0 form a lower bound of f.
=#

struct Pt
    x
    y
end

function get_sp_intersection(A, B, L)
    t = ((A.y-B.y)-L*(A.x-B.x)) / 2L;
    return Pt(A.x+t, A.y-t*L);
end

function shubert_piyavskii(f, a, b, l, ϵ, δ=0.01)
    m = (a+b)/2;
    A, M, B = Pt(a, f(a)), Pt(m, f(m)), Pt(b, f(b))
    pts = [A, get_sp_intersection(A, M, l),
    M, get_sp_intersection(M, B, l), B];
    Δ = Inf;

    while Δ > ϵ
        i = argmin([P.y for P in pts]);
        P = Pt(pts[i].x, f(pts[i].x));
        Δ = P.y = pts[i].y;

        P_prev = get_sp_intersection(pts[i-1], P, l);
        P_next = get_sp_intersection(P, pts[i+1], l);

        deleteat!(pts, i);
        insert!(pts, i, P_next);
        insert!(pts, i, P);
        insert!(pts, i, P_prev);
    end

    intervals = [];
    P_min = pts[2*(argmin([P.y for P in pts[1:2:end]])) - 1];
    y_min = P_min.y;
    for i in 2:2:length(pts)
        if pts[i].y < y_min
            dy = y_min - pts[i].y;
            x_low = max(a, pts[i].x - dy/l);
            x_high = max(b, pts[i].x - dy/l);
            if !isempty(intervals) && intervals[end][2] + δ ≥ x_low
                intervals[end] = (intervals[end][1], x_high);
            else
                push!(intervals, (x_low, x_high));
            end
        end
    end
    return (P_min, intervals);
end


### A Pluto.jl notebook ###
# v0.15.1

using Markdown
using InteractiveUtils

# ╔═╡ 61078404-224f-44f1-a75a-9b3537ecc95c
# we write a function that decomposes a square matrix A as A = LU
# where L is lower triangular and U is upper triangular
function LUdecomp(A)

    # we get the size of A
    n, m = size(A)

    # we check if A is indeed a square matrix
    if n != m
        error("Input must be a square matrix")
    end

    # we initialize the output matrices
    L = zeros(n, n)
    U = zeros(n, n)

    # we make a copy of A in order not to modify the original
    B = copy(A)

    for k = 1:n

        # we set the entries of the kth row of U
        for j = k:n
            U[k,j] = B[k,j]
        end

        # we check that we don't divide by zero
        if U[k,k] == 0
            error("** A^(k-1)_{k,k} == 0 in LU decomp")
        end

        # we set the entries of the kth column of L
        for i = k:n
           L[i,k] = B[i,k] / U[k,k]
        end

        # we modify A for the next iteration
        for i = k:n
            for j = k:n
                B[i,j] -= L[i,k] * U[k,j]
            end
        end
    end

    # we return L and U
    return (L, U)
end

# ╔═╡ 4996c2fe-068b-11ec-221d-5fdd71b0f050
md"Simple matrix for which there is no LUdecomp:"

# ╔═╡ 41b2c106-e317-4d65-bb61-94cecb4683a2
A = [0.0 2.0; 3.0 1.0];

# ╔═╡ 75d691e3-34e6-4506-b9b6-559c2412d791
md"This will cause an error:"

# ╔═╡ ffa8307d-751a-42d7-8a5d-ce8a3e92ca0d
LUdecomp(A);

# ╔═╡ a2c516ea-c23f-4f25-8e39-b5213e6fb94e
# we write a function that returns the row index of the (absolutely) largest element in column k of the matrix A
function findLargestInCol(A, k)

	# we get column k
	col = A[:,k]

	# we take elementwise absolute value
	absCol = broadcast(abs, col)

	# we get the index of the largest element
	index = argmax(absCol)

	# we return index
	return index
end

# ╔═╡ ca343c99-8fa8-4489-8c57-3e8afc1fbe71
# we write a more efficient function that swaps rows u and v of some matrix A
function swapRowsV2!(A, u, v)

	# we directly swap rows u and v of A without copying
	temp = A[u, :]
	A[u, :] = A[v, :]
	A[v, :] = temp
end

# ╔═╡ e10c36a3-d0c0-4f63-ac43-f1f04eff1e19
# we write another function that composes A into A = P^{-1}LU where P is stored as a vector and swapRows is more efficient
function PALUdecompV2(A)

    # we get the size of A
    m, n = size(A)

    # we check that A is indeed a square matrix
    if m != n
        error("Input must be a square matrix")
    end

    # we initialize L and U to zero, P to {1,2, ..., n}
    L = zeros(n, n)
    U = zeros(n, n)
    p = collect(1:n)

    # we make a copy of A to work with
    B = copy(A)

    for k = 1:n

        # we find the pivot
        maxk = findLargestInCol(B, k)
        if maxk != k
            swapRowsV2!(B, maxk, k)
            swapRowsV2!(L, maxk, k)

            # we swap two elements of P instead of two rows
            temp = p[maxk]
            p[maxk] = p[k]
            p[k] = temp
        end

        # we proceed with the normal LU decomposition algorithm
        for j = k:n
            U[k,j] = B[k,j]
        end

        # we check that we don't divide by zero
        if U[k,k] == 0
            error("** A^(k-1)_{k,k} == 0 in PALU decomp")
        end

        for i = k:n
           L[i,k] = B[i,k] / U[k,k]
        end

        for i = k:n
            for j = k:n
                B[i,j] -= L[i,k] * U[k,j];
            end
        end
    end

	P = zeros(n,n)
	for i = 1:n
		P[i,p[i]] = 1
	end

    # we return P, L and U
    return P, L, U
end

# ╔═╡ a0f23d95-18a5-427f-b52c-ebdb44df8fa3
PA, LA, UA = PALUdecompV2(A);

# ╔═╡ 23babfbd-d571-454e-8598-3fc9a9044baa
LA*UA == PA*A

# ╔═╡ a5570224-5282-48a3-8f5a-7f5f7a9b741c
md"""
Slightly less trivial example where the error appears in step $k=2$. The problem arises because $B_{22} = B_{21} B_{12} / B_{11}$ so in the usual LU decomposition $A^{1}_{22}$ is zero:
"""

# ╔═╡ 1dd12bff-9da3-4f26-ab3c-23a0e2671aa1
B = [1.0 2.0 3.0; 2.0 4.0 2.0; 1.0 3.0 2.0];

# ╔═╡ 5a34190b-5c77-4732-9134-c6bd432c053c
LUdecomp(B)

# ╔═╡ 751479d0-269c-4be3-893c-1317f9707b5f
PB, LB, UB = PALUdecompV2(B);

# ╔═╡ f5229f99-e78b-48e0-853e-ea1b50aa2633
LB*UB == PB*B

# ╔═╡ 3304fcdb-923f-4b21-932d-cdbfa1c4b243
md"Check that LUdecomp of $PB$ gives the same answer:"

# ╔═╡ b76c8925-323f-423d-98b0-a42bb7ef4f60
LUdecomp(PB*B) == (LB, UB)

# ╔═╡ 344175ec-c622-4967-8f4d-4c1507886f7a
md"Bigger matrix..."

# ╔═╡ 7bfae705-78ff-4f4b-8f85-4f623488d632
C = rand(800, 800);

# ╔═╡ 5c497b8f-d2a3-4609-bd5d-891fe21a4714
PC, LC, UC = PALUdecompV2(C);

# ╔═╡ 5c8e5b61-137e-401b-96f5-df3c9ffaa9dd
md"Estimate 'error' as sum of squares of elements of $LU-PA$:"

# ╔═╡ f76c12c0-3a8e-4ef1-b8dc-a5b2eaf8a761
sum((LC*UC-PC*C).^2)

# ╔═╡ Cell order:
# ╠═61078404-224f-44f1-a75a-9b3537ecc95c
# ╟─4996c2fe-068b-11ec-221d-5fdd71b0f050
# ╠═41b2c106-e317-4d65-bb61-94cecb4683a2
# ╟─75d691e3-34e6-4506-b9b6-559c2412d791
# ╠═ffa8307d-751a-42d7-8a5d-ce8a3e92ca0d
# ╠═a2c516ea-c23f-4f25-8e39-b5213e6fb94e
# ╠═ca343c99-8fa8-4489-8c57-3e8afc1fbe71
# ╠═e10c36a3-d0c0-4f63-ac43-f1f04eff1e19
# ╠═a0f23d95-18a5-427f-b52c-ebdb44df8fa3
# ╠═23babfbd-d571-454e-8598-3fc9a9044baa
# ╟─a5570224-5282-48a3-8f5a-7f5f7a9b741c
# ╠═1dd12bff-9da3-4f26-ab3c-23a0e2671aa1
# ╠═5a34190b-5c77-4732-9134-c6bd432c053c
# ╠═f5229f99-e78b-48e0-853e-ea1b50aa2633
# ╠═751479d0-269c-4be3-893c-1317f9707b5f
# ╟─3304fcdb-923f-4b21-932d-cdbfa1c4b243
# ╠═b76c8925-323f-423d-98b0-a42bb7ef4f60
# ╟─344175ec-c622-4967-8f4d-4c1507886f7a
# ╠═7bfae705-78ff-4f4b-8f85-4f623488d632
# ╠═5c497b8f-d2a3-4609-bd5d-891fe21a4714
# ╟─5c8e5b61-137e-401b-96f5-df3c9ffaa9dd
# ╠═f76c12c0-3a8e-4ef1-b8dc-a5b2eaf8a761


#################### Basis for Polynomial Space ####################3
struct PolynomialBasis{T}
  order::Int
  nodes::Vector{T}      #Quadrature nodes for numerical integration
  weights::Vector{T}    #Quadrature weights for numerical integration
  polynomials::Vector{Poly}
  φ::Matrix{T}     #Vandermonde matrix: basis polynomials evaluated on G-L nodes
  dφ::Matrix{T}
  invφ::Matrix{T}   #inverse of Gen. Vandermonde matrix
end

"""compute Legendre polynomials coefficients, normalized to be orthonormal"""
function poly_legendre{T<:Number}(n, ::Type{T}=Float64, var=:x)
    return poly_jacobi(n,0.0,0.0,T,var)
end

"""compute Jacobi polynomials coefficients, normalized to be orthonormal"""
function poly_jacobi{T<:Number}(n, a, b, ::Type{T}=Float64, var=:x)
    ox = one(T)
    zx = zero(T)
    #Compute initial P_0 and P_1
    γ0 = 2^(a+b+1)/(a+b+1)*gamma(a+1)*gamma(b+1)/gamma(a+b+1);
    p0 = Poly{T}([one(T)/sqrt(γ0)], var)
    if n==0; return p0; end
    γ1 = (a+1)*(b+1)/(a+b+3)*γ0
    p1 = Poly{T}([(a-b)/2/sqrt(γ1), (a+b+2)/2/sqrt(γ1)], var)
    if n==1; return p1; end
    px = Poly{T}([zero(T), one(T)], var)
    aold = 2/(2+a+b)*sqrt((a+1)*(b+1)/(a+b+3))
    for i = 1:(n-1)
        h1 = 2*i+a+b;
        anew = 2/(h1+2)*sqrt((i+1)*(i+1+a+b)*(i+1+a)*(i+1+b)/(h1+1)/(h1+3))
        bnew = -(a^2-b^2)/h1/(h1+2);
        p2 = ox/anew*(-aold*p0 + (px-bnew)*p1);
        aold =anew;
        p0 = p1
        p1 = p2
    end
    return p1
end

function legendre_basis{T<:Number}(order, ::Type{T}=Float64)
  nodes, weights = gausslobatto(order+1)
  φ = zeros(T,order+1,order+1)
  dφ = zeros(T,order+1,order+1)
  polynomials = Vector{Poly}(order+1)
  for n = 0:order
    p = poly_legendre(n, T)
    dp = polyder(p)
    polynomials[n+1] = p
    # Eval interior nodes
    φ[:,n+1] = polyval(p, nodes)
    dφ[:,n+1] = polyval(dp, nodes)
  end
  invφ = inv(φ)
  PolynomialBasis{T}(order,nodes,weights,polynomials,φ,dφ,invφ)
end

"Maps reference coordinates (ξ ∈ [-1,1]) to interval coordinates (x)"
function reference_to_interval(ξ,a::Tuple)
   0.5*(a[2]-a[1])*ξ + 0.5*(a[2]+a[1])
end

"Project function f on polynomial space Vₕ"
function project_function(f, basis, interval::Tuple; component=1)
  nodes = reference_to_interval(basis.nodes, interval)
  f_val = zeros(nodes)
  for i in 1:size(nodes,1)
    f_val[i] = f(nodes[i])[component]
  end
  function model(x,p)
    result = zeros(x)
    for i in 1:size(p,1)
      result.+=p[i]*polyval(basis.polynomials[i], x)
    end
    result
  end
  p0 = zeros(eltype(basis.nodes),basis.order+1)
  curve_fit(model, basis.nodes, f_val, p0)
end

# using FEMBasis
# import FEMBasis: create_basis
# a = legendre_basis(2)
#
# code = create_basis(
# :LegSeg3,
# "3 node linear segment/line orthogonal legendre element",
# NTuple((
#  (-0.7745966692414834,), # N1
#  ( 0.0, ), # N2
#  ( 0.7745966692414834, ), # N3
# )),(
#  :(0.5 + 0.5),
#  :(1.0*u),
#  :(-0.5 + 1.5*u^2),)
# )
# eval(code)
#
# eval_basis!(LegSeg3(),zeros(3,1),(-0.7745966692414834))
# eval_dbasis!(LegSeg3(),zeros(1,3),(-0.7745966692414834))


# # Polynomial Optimization

#md # [![](https://mybinder.org/badge_logo.svg)](@__BINDER_ROOT_URL__/generated/Polynomial Optimization/polynomial_optimization.ipynb)
#md # [![](https://img.shields.io/badge/show-nbviewer-579ACA.svg)](@__NBVIEWER_ROOT_URL__/generated/Polynomial Optimization/polynomial_optimization.ipynb)
# **Contributed by**: Benoît Legat

# ## Introduction

# Consider the polynomial optimization problem [L09, Example 2.2] of
# minimizing the polynomial $x^3 - x^2 + 2xy - y^2 + y^3$
# over the polyhedron defined by the inequalities $x \ge 0, y \ge 0$ and $x + y \geq 1$.

# [L09] Lasserre, J. B.
# *Moments, positive polynomials and their applications*.
# World Scientific, **2009**.

using Test #src
using DynamicPolynomials
@polyvar x y
p = x^3 - x^2 + 2x*y -y^2 + y^3
using SumOfSquares
S = @set x >= 0 && y >= 0 && x + y >= 1
p(x=>1, y=>0), p(x=>1//2, y=>1//2), p(x=>0, y=>1)

# The optimal solutions are $(x, y) = (1, 0)$ and $(x, y) = (0, 1)$ with objective value $0$ but [Ipopt](https://github.com/jump-dev/Ipopt.jl/) only finds the local minimum $(1/2, 1/2)$ with objective value $1/4$.

import Ipopt
model = Model(optimizer_with_attributes(Ipopt.Optimizer, "print_level" => 0))
@variable(model, a >= 0)
@variable(model, b >= 0)
@constraint(model, a + b >= 1)
@NLobjective(model, Min, a^3 - a^2 + 2a*b - b^2 + b^3)
optimize!(model)
@test termination_status(model) == MOI.LOCALLY_SOLVED #src
@show termination_status(model)
@test value(a) ≈ 0.5 rtol=1e-5 #src
@show value(a)
@test value(b) ≈ 0.5 rtol=1e-5 #src
@show value(b)
@test objective_value(model) ≈ 0.25 rtol=1e-5 #src
@show objective_value(model)

# Note that the problem can be written equivalently as follows using [registered functions](https://jump.dev/JuMP.jl/v0.21/nlp/#User-defined-Functions-1).

using Ipopt
model = Model(optimizer_with_attributes(Ipopt.Optimizer, "print_level" => 0))
@variable(model, a >= 0)
@variable(model, b >= 0)
@constraint(model, a + b >= 1)
peval(a, b) = p(x=>a, y=>b)
register(model, :peval, 2, peval, autodiff=true)
@NLobjective(model, Min, peval(a, b))
optimize!(model)
@test termination_status(model) == MOI.LOCALLY_SOLVED #src
@show termination_status(model)
@test value(a) ≈ 0.5 rtol=1e-5 #src
@show value(a)
@test value(b) ≈ 0.5 rtol=1e-5 #src
@show value(b)
@test objective_value(model) ≈ 0.25 rtol=1e-5 #src
@show objective_value(model)

## Sum-of-Squares approach

# We will now see how to find the optimal solution using Sum of Squares Programming.
# We first need to pick an SDP solver, see [here](https://jump.dev/JuMP.jl/v0.21.6/installation/#Supported-solvers) for a list of the available choices.

import CSDP
solver = optimizer_with_attributes(CSDP.Optimizer, MOI.Silent() => true)

# A Sum-of-Squares certificate that $p \ge \alpha$ over the domain `S`, ensures that $\alpha$ is a lower bound to the polynomial optimization problem.
# The following program searches for the largest upper bound and finds zero.

model = SOSModel(solver)
@variable(model, α)
@objective(model, Max, α)
@constraint(model, c3, p >= α, domain = S)
optimize!(model)
@test termination_status(model) == MOI.OPTIMAL #src
@show termination_status(model)
@test objective_value(model) ≈ 0.0 atol=1e-5 #src
@show objective_value(model)

# Using the solution $(1/2, 1/2)$ found by Ipopt of objective value $1/4$
# and this certificate of lower bound $0$ we know that the optimal objective value is in the interval $[0, 1/4]$
# but we still do not know what it is (if we consider that we did not try the solutions $(1, 0)$ and $(0, 1)$ as done in the introduction).
# If the dual of the constraint `c3` was atomic, its atoms would have given optimal solutions of objective value $0$ but that is not the case.

ν3 = moment_matrix(c3)
@test extractatoms(ν3, 1e-3) === nothing #src
extractatoms(ν3, 1e-3) # Returns nothing as the dual is not atomic

# Fortunately, there is a hierarchy of programs with increasingly better bounds that can be solved until we get one with atom dual variables.
# This comes from the way the Sum-of-Squares constraint with domain `S` is formulated.
# The polynomial $p - \alpha$ is guaranteed to be nonnegative over the domain `S` if there exists Sum-of-Squares polynomials $s_0$, $s_1$, $s_2$, $s_3$ such that
# ```math
# p - \alpha = s_0 + s_1 x + s_2 y + s_3 (x + y - 1).
# ```
# Indeed, in the domain `S`, $x$, $y$ and $x + y - 1$ are nonnegative so the right-hand side is a sum of squares hence is nonnegative.
# Once the degrees of $s_1$, $s_2$ and $s_3$ have been decided, the degree needed for $s_0$ will be determined but we have a freedom in choosing the degrees of $s_1$, $s_2$ and $s_3$.
# By default, they are chosen so that the degrees of $s_1 x$, $s_2 y$ and $s_3 (x + y - 1)$ match those of $p - \alpha$ but this can be overwritten using the `maxdegree` keyword argument.

# ### The maxdegree keyword argument

# The maximum total degree (i.e. maximum sum of the exponents of $x$ and $y$) of the monomials of $p$ is 3 so the constraint in the program above is equivalent to `@constraint(model, p >= α, domain = S, maxdegree = 3)`.
# That is, since $x$, $y$ and $x + y - 1$ have total degree 1, the sum of squares polynomials $s_1$, $s_2$ and $s_3$ have been chosen with maximum total degree $2$.
# Since these polynomials are sums of squares, their degree must be even so the next maximum total degree to try is 4.
# For this reason, the keywords `maxdegree = 4` and `maxdegree = 5` have the same effect in this example.
# In general, if the polynomials in the domain are not all odd or all even, each value of `maxdegree` has a different effect in the choice of the maximum total degree of some $s_i$.

model = SOSModel(solver)
@variable(model, α)
@objective(model, Max, α)
@constraint(model, c5, p >= α, domain = S, maxdegree = 5)
optimize!(model)
@test termination_status(model) == MOI.OPTIMAL #src
@show termination_status(model)
@test objective_value(model) ≈ 0.0 atol=1e-5 #src
@show objective_value(model)

# As shown below, for `maxdegree = 5`, the dual variable is atomic as it is the moments of the measure
# $$0.5 \delta(x-1, y) + 0.5 \delta(x, y-1)$$
# where $\delta(x, y)$ is the dirac measure centered at $(0, 0)$.
# Therefore the program provides both a certificate that $0$ is a lower bound and a certificate that it is also an upper bound since it is attained at the global minimizers $(1, 0)$ and $(0, 1)$.

ν5 = moment_matrix(c5)
atoms5 = extractatoms(ν5, 1e-3) #src
@test atoms5.atoms[1].weight ≈ 0.5 rtol=1e-2 #src
@test atoms5.atoms[2].weight ≈ 0.5 rtol=1e-2 #src
@test atoms5.atoms[1].center[2:-1:1] ≈ atoms5.atoms[2].center[1:2] rtol=1e-2 #src
extractatoms(ν5, 1e-3)

# ## A deeper look into atom extraction

# The `extractatoms` function requires a `ranktol` argument that we have set to `1e-3` in the preceding section.
# This argument is used to transform the dual variable into a system of polynomials equations whose solutions give the atoms.
# This transformation uses the SVD decomposition of the moment matrix and discards the equations corresponding to a singular value lower than `ranktol`.
# When this system of equation has an infinite number of solutions, `extractatoms` concludes that the measure is not atomic.
# For instance, with `maxdegree = 3`, we obtain the system
# $$x + y = 1$$
# which contains a whole line of solution.
# This explains `extractatoms` returned `nothing`.

ν3 = moment_matrix(c3)
SumOfSquares.MultivariateMoments.computesupport!(ν3, 1e-3)
@test length(ν3.support.I.p) == 1 #src

# With `maxdegree = 5`, we obtain the system
# ```math
# \begin{aligned}
#   x + y & = 1\\
#   y^2 & = y\\
#   xy & = 0\\
#   x^2 + y & = 1
# \end{aligned}
# ```

ν5 = moment_matrix(c5)
SumOfSquares.MultivariateMoments.computesupport!(ν5, 1e-3)

# This system can be reduced to the equivalent system
# ```math
# \begin{aligned}
#   x + y & = 1\\
#   y^2 & = y
# \end{aligned}
# ```
# which has the solutions $(0, 1)$ and $(1, 0)$.

SemialgebraicSets.computegröbnerbasis!(ideal(ν5.support))
ν5.support
@test length(ν5.support.I.p) == 2 #src

# The function `extractatoms` then reuses the matrix of moments to find the weights $1/2$, $1/2$ corresponding to the diracs centered respectively at $(0, 1)$ and $(1, 0)$.
# This details how the function obtained the result
# $$0.5 \delta(x-1, y) + 0.5 \delta(x, y-1)$$
# given in the previous section.

# ## HomotopyContinuation

# As discussed in the previous section, the atom extraction relies on the solution
# of a system of algebraic equations. The `extractatoms` function takes an optional
# `solver` argument that is used to solve this system of equation.
# If no solver is provided, the default solver of SemialgebraicSets.jl is used which
# currently computes the Gröbner basis, then the multiplication matrices and
# then the Schur decomposition of a random combination of these matrices.
# As the system of equations is obtained from a numerical solution and is represented
# using floating point coefficients, homotopy continuation is recommended as it is
# more numerically robust than Gröbner basis computation.
# The following uses homotopy continuation to solve the system of equations.

using HomotopyContinuation
solver = SemialgebraicSetsHCSolver(; excess_residual_tol = 2e-2, real_tol = 2e-2, compile = false)
atoms5 = extractatoms(ν5, 1e-3, solver) #src
@test length(atoms5.atoms) == 2 #src
@test atoms5.atoms[1].weight + atoms5.atoms[2].weight ≈ 1.0 rtol=1e-2 #src
@test atoms5.atoms[1].center[2:-1:1] ≈ atoms5.atoms[2].center[1:2] rtol=1e-2 #src
extractatoms(ν5, 1e-3, solver)

# As the system has 3 equations for 2 variables and the coefficients of the equations
# are to be treated with tolerance since they originate from the solution of an SDP,
# we need to set `excess_residual_tol` and `real_tol` to a high tolerance otherwise,
# HomotopyContinuation would consider that there is no solution.
# Indeed, as the system is overdetermined (it has more equations than variables)
# HomotopyContinuation expects to have excess solution hence it filters out
# excess solution among the solution found. It determines which solution are in excess
# by comparing the infinity norm of the residuals of the equations at the solution with `excess_residual_tol`.
# It also filters out solution for which the absolute value of the imaginary part of one of the entry
# is larger than `real_tol` and strips out the imaginary part.
# The raw solutions obtained by HomotopyContinuation can be obtained as follows:

F = HomotopyContinuation.System(ν5.support)
res = HomotopyContinuation.solve(F, solver.options...)
r = path_results(res) #src
@test length(r) == 4 #src
@test all(HomotopyContinuation.is_excess_solution, r) #src
path_results(res)

# The printed `residual` above shows why `2e-2` allows to filter how the 2 actual
# solutions from the 2 excess solutions.


using Distributions

include("confidence_interval.jl")

function one_prop_z_int(p, n, lvl)
    distribution = Normal()

    standard_error = sqrt((p * (1-p)) / n)
    critical_value = quantile(distribution, lvl + (1 - lvl) / 2)

    lowest_value = round(p - critical_value * standard_error, 4)
    highest_value = round(p + critical_value * standard_error, 4)

    return ConfidenceInterval(lowest_value, highest_value)
end

function one_prop_z_test(p, p0, n, alpha, alt)
    distribution = Normal(p0, sqrt((p0 * (1-p0)) / n))

    if alt == "<"
        return round(cdf(distribution, p), 4)
    elseif alt == ">"
        return round(1 - cdf(distribution, p), 4)
    elseif alt == "!="
        if p > p0
            return round(2 * (1 - cdf(distribution, p)), 4)
        else
            return round(2 * cdf(distribution, p), 4)
        end
    else
        return -1
    end
end

function two_prop_z_test(p1, n1, p2, n2, alpha, alt)
    p_pooled = (p1*n1 + p2*n2) / (n1 + n2)
    p_diff = p1 - p2
    distribution = Normal(0, sqrt((p_pooled*(1-p_pooled)/n1) + (p_pooled*(1-p_pooled)/n2)))

    if alt == "<"
        return round(cdf(distribution, p_diff), 4)
    elseif alt == ">"
        return round(1 - cdf(distribution, p_diff), 4)
    elseif alt == "!="
        if p_diff > 0
            return round(2 * (1 - cdf(distribution, p_diff)), 4)
        else
            return round(2 * cdf(distribution, p_diff), 4)
        end
    else
        return -1
    end
end

function two_prop_z_int(p1, n1, p2, n2, lvl)
    standard_error = sqrt(((p1 * (1-p1)) / n1) + ((p2 * (1-p2)) / n2))

    standard_normal = Normal()
    critical_value = quantile(standard_normal, lvl + (1 - lvl) / 2)

    lowest_value = round((p1-p2) - critical_value * standard_error, 4)
    highest_value = round((p1-p2) + critical_value * standard_error, 4)

    return ConfidenceInterval(lowest_value, highest_value)
end

function z_interval(x̄, sx, n, lvl)
    distribution = Normal()

    if n < 30
        println("Conditions weren't met, using t-distribution...\n")
        distribution = TDist(n - 1)
    end

    standard_error = sx / sqrt(n)
    critical_value = quantile(distribution, lvl + (1 - lvl) / 2)

    lowest_value = round(x̄ - critical_value * standard_error, 4)
    highest_value = round(x̄ + critical_value * standard_error, 4)

    return ConfidenceInterval(lowest_value, highest_value)
end

function z_test(x̄, μ0, sx, n, alpha, alt)
    distribution = Normal(μ0, sx / sqrt(n))

    if alt == "<"
        return round(cdf(distribution, x̄), 4)
    elseif alt == ">"
        return round(1 - cdf(distribution, x̄), 4)
    elseif alt == "!="
        if x̄ > μ0
            return round(2 * (1 - cdf(distribution, x̄)), 4)
        else
            return round(2 * cdf(distribution, x̄), 4)
        end
    else
        return -1
    end
end

function two_sample_z_interval(x̄1, sx1, n1, x̄2, sx2, n2, lvl)
    distribution = Normal()

    if n1 < 30 || n2 < 30
        println("Conditions weren't met, using t-distribution...\n")
        distribution = TDist(n1 + n2 - 2)
    end

    standard_error = sqrt((((n1 - 1) * sx1 ^ 2) + ((n2 - 1) * sx2 ^ 2)) / (n1 + n2 - 2)) * sqrt(1 / n1 + 1 / n2)
    critical_value = quantile(distribution, lvl + (1 - lvl) / 2)

    lowest_value = round(x̄1 - x̄2 - critical_value * standard_error, 4)
    highest_value = round(x̄1 - x̄2 + critical_value * standard_error, 4)

    return ConfidenceInterval(lowest_value, highest_value)
end

function two_sample_z_test(x̄1, sx1, n1, x̄2, sx2, n2, lvl, alt)
    z = (x̄1 - x̄2) / sqrt((sx1^2 / n1 + sx2^2 / n2))

    standard_normal = Normal()

    if alt == "<"
        return round(cdf(standard_normal, abs(z)), 4)
    elseif alt == ">"
        return round(1 - cdf(standard_normal, abs(z)), 4)
    elseif alt == "!="
        return round(2 * (1 - cdf(standard_normal, abs(z))), 4)
    else
        return -1
    end
end


function integrate_wrt_normal_pdf(f::Function, μ::Real, σ::Real; npts::Int=15)
    nodes, wts = gausshermite(npts)
    return 1/sqrtπ * sum( f.(sqrt2 .* σ .* nodes .+ μ) .* wts)
end

function gh_chg_of_vars(f, ndswts::NTuple{N}, μ, L) where {N}
    nodes = SVector{N}(first.(ndswts)...)
    wt = prod(last.(ndswts))
    feval = f(sqrt2 .* L*nodes .+ μ)
    return wt*feval
end

function integrate_wrt_normal_pdf(f::Function, μ::AbstractVector, L::AbstractMatrix; npts::Int=15)
    k = length(μ)
    k == checksquare(L) || throw(DomainError())

    ndswts = zip(gausshermite(npts)...)

    # https://stackoverflow.com/questions/56120583/n-dimensional-cartesian-product-of-a-set-in-julia
    quadprod = product( ntuple(i->ndswts, k)...)

    #closure
    g(node_wt_ntup) = gh_chg_of_vars(f, node_wt_ntup, μ, L)

    return (1/sqrtπ)^k * sum( g(vw) for vw in quadprod)
end


# Generate the first n numbers in Halton's sequence with base b.
#
#   seq = haltonsequence(b, n)
#
# Output:
#  seq - sequence of n Halton numbers
#
# Inputs:
#  b - base of the sequence
#  n - length of vector to be generated
#
function haltonsequence(b, n)
  map((i) -> haltonnumber(b, i), 1:n)
end

# Generate the nth Halton number in the sequence with base b.
#
# Notes:
#  Implementation is based on the psudo code in:
#    http://en.wikipedia.org/wiki/Halton_sequence
#
function haltonnumber(base, index)
  res = 0
  f = 1 / base
  i = index

  while (i > 0)
    res += f * (i % base)
    i = ifloor(i / base)
    f = f / base
  end

  return res
end


### A Pluto.jl notebook ###
# v0.12.11

using Markdown
using InteractiveUtils

# ╔═╡ cb23b3e4-30ba-11eb-048b-33940fa83826
using ModelingToolkit

# ╔═╡ 23f048f2-30bb-11eb-2d54-9fb1c6aed0a5
using Latexify

# ╔═╡ 0d2468ee-30be-11eb-0d75-ed21f3712b85
using Plots

# ╔═╡ 44e05d48-30ba-11eb-1820-c92042f7fc1f
md"ModelingToolkit.jl is a modeling language for high-performance symbolic-numeric computation in scientific computing and scientific machine learning. It allows for users to give a high-level description of a model for symbolic preprocessing to analyze and enhance the model. ModelingToolkit can automatically generate fast functions for model components like Jacobians and Hessians, along with automatically sparsifying and parallelizing the computations. Automatic transformations, such as index reduction, can be applied to the model to make it easier for numerical solvers to handle.
"

# ╔═╡ f6b027e8-30ba-11eb-3279-cfb2ba45c50b
@variables x y

# ╔═╡ 0a1f16ba-30bb-11eb-2116-218fecf8c8e5
z = x^3 + y

# ╔═╡ 10315162-30bb-11eb-1462-d3fb963a29ec
latexify(z)

# ╔═╡ 0887b66c-30bc-11eb-0353-93ba2f2fcba1
z1 = t + t^2

# ╔═╡ 2f5af0ba-30bc-11eb-21d7-43f31111d22e
@derivatives D'~t

# ╔═╡ 1ccf0eae-30bc-11eb-1d8a-cd7f727bb69b
D(z1)

# ╔═╡ 40072080-30bc-11eb-2b27-fb501cc055c8
dz1=expand_derivatives(D(z1))

# ╔═╡ 55a84814-30bc-11eb-25df-cff76501368c
latexify(z1)

# ╔═╡ 4d6c6c00-30bc-11eb-1da6-216840f18165
latexify(dz1)

# ╔═╡ c710d79e-30bc-11eb-06df-8f322f48cda0
fz1=build_function(z1,t,expression=false)

# ╔═╡ 881a68a6-30bd-11eb-35c2-4dc061e74ce2
t1=[1:20...]

# ╔═╡ 1b06d722-30bd-11eb-0bbc-27e45fde5a46
ft1=fz1.([1:20...])

# ╔═╡ f8f7d7c2-30bd-11eb-13ee-e9e682866368
plot(t1,ft1)

# ╔═╡ 9e94b908-30bb-11eb-19f0-33840780b3ee
build_function(z,[x,y])

# ╔═╡ a41d0c4a-30bb-11eb-1434-4982edcff617

# ╔═╡ 440889ec-30bb-11eb-3c5c-a519f291111a
A = [x^2+y 0 2x
     0     0 2y
     y^2+x 0 0]

# ╔═╡ 4b7b92b2-30bb-11eb-342f-0fb5b3f27ec5
latexify(A)

# ╔═╡ Cell order:
# ╠═44e05d48-30ba-11eb-1820-c92042f7fc1f
# ╠═cb23b3e4-30ba-11eb-048b-33940fa83826
# ╠═23f048f2-30bb-11eb-2d54-9fb1c6aed0a5
# ╠═0d2468ee-30be-11eb-0d75-ed21f3712b85
# ╠═f6b027e8-30ba-11eb-3279-cfb2ba45c50b
# ╠═0a1f16ba-30bb-11eb-2116-218fecf8c8e5
# ╠═10315162-30bb-11eb-1462-d3fb963a29ec
# ╠═0887b66c-30bc-11eb-0353-93ba2f2fcba1
# ╠═2f5af0ba-30bc-11eb-21d7-43f31111d22e
# ╠═1ccf0eae-30bc-11eb-1d8a-cd7f727bb69b
# ╠═40072080-30bc-11eb-2b27-fb501cc055c8
# ╠═55a84814-30bc-11eb-25df-cff76501368c
# ╠═4d6c6c00-30bc-11eb-1da6-216840f18165
# ╠═c710d79e-30bc-11eb-06df-8f322f48cda0
# ╠═881a68a6-30bd-11eb-35c2-4dc061e74ce2
# ╠═1b06d722-30bd-11eb-0bbc-27e45fde5a46
# ╠═f8f7d7c2-30bd-11eb-13ee-e9e682866368
# ╠═9e94b908-30bb-11eb-19f0-33840780b3ee
# ╠═a41d0c4a-30bb-11eb-1434-4982edcff617
# ╠═440889ec-30bb-11eb-3c5c-a519f291111a
# ╠═4b7b92b2-30bb-11eb-342f-0fb5b3f27ec5


export Tanh

"""
`Tanh([domainType=Float64::Type,] dim_in::Tuple)`

Creates an hyperbolic tangent non-linear operator with input dimensions `dim_in`:
```math
\\text{tanh} ( \\mathbf{x} ).
```

"""
struct Tanh{T,N} <: NonLinearOperator
	dim::NTuple{N,Int}
end

function Tanh(DomainType::Type, DomainDim::NTuple{N,Int}) where {N}
	Tanh{DomainType,N}(DomainDim)
end

Tanh(DomainDim::NTuple{N,Int}) where {N} = Tanh{Float64,N}(DomainDim)
Tanh(DomainDim::Vararg{Int}) = Tanh{Float64,length(DomainDim)}(DomainDim)

function mul!(y::AbstractArray{T,N}, L::Tanh{T,N}, x::AbstractArray{T,N}) where {T,N}
	y .= tanh.(x)
end

function mul!(y::AbstractArray,
              J::AdjointOperator{Jacobian{A,TT}},
              b::AbstractArray) where {T,N, A<: Tanh{T,N}, TT <: AbstractArray{T,N}}
    L = J.A
    y .= conj.(sech.(L.x).^2).*b
end

fun_name(L::Tanh) = "tanh"

size(L::Tanh) = (L.dim, L.dim)

domainType(L::Tanh{T,N}) where {T,N} = T
codomainType(L::Tanh{T,N}) where {T,N} = T


function fixedpointmap(f, x_0, tolerance, maxiter)
    D(f) = x -> ForwardDiff.derivative(f, x)
    f_prime = D(f)
    x_old = x_0
    normdiff = Inf
    iter = 1
    while normdiff > tolerance && iter <= maxiter
        x_new = x_old - f(x_old)/f_prime(x_old)
        normdiff = norm(x_new - x_old)
        x_old = x_new
        iter = iter + 1
    end
    return (value = x_old, normdiff = normdiff, iter = iter)
end

# define a map and parameters
f(x) = (x-1)^2

sol = fixedpointmap(f, 0, 1.0E-8, 1000)
println("Fixed point = $(sol.value), and |x_t+1 - x_t| = $(sol.normdiff) in $(sol.iter) iterations")


"""
2520 is the smallest number that can be divided by each of the numbers from 1 to 10 without any remainder.

What is the smallest positive number that is evenly divisible by all of the numbers from 1 to 20?
"""

ismultiple(x::Int,r::UnitRange) = all([x%i == 0 for i in r])

function smallestmutiple(nmin,nmax)
    r = collect(nmin:nmax)
    multiples = []

    for i in eachindex(r)   # store only the common divisor
        t = r[i]
        t == 1 && continue
        push!(multiples,r[i])

        for j in i:length(r)
            if r[j] % t == 0
                r[j] = r[j] / t
            end
        end
    end
    return prod(multiples)
end

@assert smallestmutiple(1,10) == 2520
@show smallestmutiple(1,20)


# ------------------------------------------------------------------
# Licensed under the MIT License. See LICENSE in the project root.
# ------------------------------------------------------------------

"""
    signarea(A, B, C)

Compute signed area of triangle formed
by points `A`, `B` and `C`.
"""
function signarea(A::Point{2}, B::Point{2}, C::Point{2})
  ((B - A) × (C - A)) / 2
end

"""
    sideof(point, segment)

Determines on which side of the oriented `segment`
the `point` lies. Possible results are `:LEFT`,
`:RIGHT` or `:ON` the segment.
"""
function sideof(p::Point{2,T}, s::Segment{2,T}) where {T}
  a, b = vertices(s)
  area = signarea(p, a, b)
  ifelse(area > atol(T), :LEFT, ifelse(area < -atol(T), :RIGHT, :ON))
end

"""
    sideof(point, chain)

Determines on which side of the closed `chain` the
`point` lies. Possible results are `:INSIDE` or
`:OUTSIDE` the chain.
"""
function sideof(p::Point{2,T}, c::Chain{2,T}) where {T}
  w = windingnumber(p, c)
  ifelse(isapprox(w, zero(T), atol=atol(T)), :OUTSIDE, :INSIDE)
end