"""

Contains embedding of data under diffusion maps.
"""
struct DiffMap
    Y  # m x n
    L
    Λ
    Φ
end

"""

"""
function DiffMap(X; m::Integer=2, σ = 2)
    D, n = size(X)

    # # Step 1: Compute similarity matrix
    𝒦 = gaussian_similarity(X, σ)

    # Step 2: compute laplacian
    L = rlap(𝒦)

    # Step 3: Compute eigenvectors of L
    # Arpack returns complex-valued eigenvalues.
    Λ, Φ = eigs(L, nev=m+1, which=:SR)
    Λ = real(Λ)
    Φ = real(Φ)

    # Remember that (1,...,1) is an eigenvector with eigenvalue 0. This is uninformative,
    # hence we discard it as usual.
    Φ ./= Φ[:, 1]
    Φ = Φ[:, 2:m+1]
    Λ = Λ[2:m+1]
    Y = permutedims(Φ .* reshape(Λ, (1, :)))
    return DiffMap(Y, L, Λ, Φ)
end

getm(DM::DiffMap) = size(DM.Y, 1)
getn(DM::DiffMap) = size(DM.Y, 2)
size(DM::DiffMap) = size(DM.Y)
size(DM::DiffMap, d::Integer) = size(DM.Y, d)

"""
    Return measure of density for laplacian. (#nonzeros / #total)
"""
sparsity(DM::DiffMap) = sparsity(DM.L)
sparsity(A) = sum(iszero.(A)) / length(A)

function Base.iterate(DM::DiffMap, state = 1)
    if state > 4
        return nothing
    else
        getfield(DM, state), state + 1
    end
end


# # Logistic regression

import MaximumLikelihoodProblems

import Distributions
import ForwardDiff
import LogDensityProblems
import StatsFuns
import TransformVariables

struct LogisticRegression{Ty, TX}
    y::Ty
    X::TX
end

function (problem::LogisticRegression)(θ)
    y = problem.y
    X = problem.X

    β = θ.β

    η = X * β
    μ = StatsFuns.logistic.(η)
    log_likelihood = sum(Distributions.logpdf.(Distributions.Bernoulli.(μ), y))
    return log_likelihood
end

N = 10_000

## the first column (the column of all ones) is the intercept
X = hcat(ones(N), randn(N))

size_β = (2,)
β_true = [1.0, 2.0]
η_true = X * β_true
μ_true = StatsFuns.logistic.(η_true)
y = rand.(Distributions.Bernoulli.(μ_true))

problem = LogisticRegression(y, X)
transformation = TransformVariables.as((β = TransformVariables.as(Array, size_β), ))
transformed_problem = LogDensityProblems.TransformedLogDensity(transformation,
                                                               problem)
transformed_gradient_problem = LogDensityProblems.ADgradient(:ForwardDiff,
                                                             transformed_problem)

β_hat_initial_guess = zeros(size_β)
θ_hat_initial = (; β = β_hat_initial_guess)

# θ_hat:

θ_hat = MaximumLikelihoodProblems.fit(transformed_gradient_problem,
                                      θ_hat_initial)

# β_hat:

β_hat = θ_hat[:β]

# Value of the log likelihood function evaluated at θ_hat:

MaximumLikelihoodProblems.loglikelihood(transformed_gradient_problem, θ_hat)


# My formulation

#=
@sym begin
  LagrangePolynomial(xi_, yi_) := Module([N, sum, num, den],
    begin
      N = Length(xi)
      sum = 0
      For( i=1, i <= N, Increment(i),
        begin
          num = 1.0
          den = 1.0
          For( j=1, j <= N, Increment(j),
            begin
              num = If(j != i, num = num * (x-xi[j]), num)
              den = If(j != i, den = den * (xi[i]-xi[j]), den)
            end
          )
          sum = sum + yi[i] * num/den
        end
      ),
      Return(Simplify(sum))
    end
  )
end
=#

# John's better formulation, much better, need to update tests

@sym begin
  LagrangePolynomial(xi_, yi_) := Module([N, sum, num, den],
    begin
      N = Length(xi)
      sum = 0
      For( i=1, i <= N, Increment(i),
        begin
          num = 1
          den = 1
          For( j=1, j <= N, Increment(j),
            begin
              If(j != i, num *= (x-xi[j]))
              If(j != i, den *= (xi[i]-xi[j]))
            end
          )
          sum += yi[i] * num/den
        end
      ),
      Return(Simplify(sum))
    end
  )
end


using PyPlot, PyCall

function euler(f, y0, h, N, t0=0.0)
    t = t0 .+ h*(0:N)
    y = zeros(length(y0), N+1)

    y[:,1] .= y0
    for n = 1:N
        y[:,n+1] = y[:,n] + h * f(t[n], y[:,n])
    end

    return t,y
end

f(t,y) = -y .+ sin(t)
t,y = euler(f, 1, 0.2, 20)
plot(t, y[:], "-o");

yexact(t) = exp(-t) + (sin(t) - cos(t) + exp(-t)) / 2
tt = 0:0.01:2
plot(tt, yexact.(tt))
for h = [0.5, 0.2, 0.1]
    t,y = euler(f, 1, h, round(Int, 2/h))
    plot(t, y[:], "-o")
end
legend(("Exact", "h=0.5", "h=0.2", "h=0.1"));

function rk4(f, y0, h, N, t0=0)
    t = t0 .+ h*(0:N)
    y = zeros(length(y0), N+1)

    y[:,1] .= y0
    for n = 1:N
        k1 = h * f(t[n], y[:,n])
        k2 = h * f(t[n] + h/2, y[:,n] + k1/2)
        k3 = h * f(t[n] + h/2, y[:,n] + k2/2)
        k4 = h * f(t[n] + h, y[:,n] + k3)
        y[:,n+1] = y[:,n] + (k1 + 2k2 + 2k3 + k4) / 6
    end

    return t,y
end

yexact(t) = exp(-t) + (sin(t) - cos(t) + exp(-t)) / 2
tt = 0:0.01:2
plot(tt, yexact.(tt))
for h = [0.5, 0.2, 0.1]
    t,y = rk4(f, 1, h, round(Int, 2/h))
    plot(t, y[:], "-o")
end
legend(("Exact", "h=0.5", "h=0.2", "h=0.1"));

errors = @. y[:] - yexact(t[:])
max_error = maximum(abs.(errors))


module Quadrature
using RandomNumbers

# Functions we will integrate
f1(x) = (1.0 / sqrt(2 * pi)) * exp((-x^2.0) / 2.0)
f1lim = (0.0, 1.0)

f2(x) = ((sin(2.0 * x))^2.0)/(x^2.0)
f2lim = (pi, 2 * pi)

f3(x) = x^(-.5)
f3lim = (0.0, 2.0)

# Change of variable for f1 -> Inf
f4(x) = f1(-log(x)) / x
f4lim = (0.0, e^-f1lim[1])

# Infinite limits of integration for printing
f1inflim = (0.0, Inf)

# PDF of the normal distribution
function normpdf(x, mu=0, sigma=1)
	return (sigma * sqrt(2 * pi))^-1.0 * e^(-(x - mu)^2 / (2 * sigma^2))
end

# Limits to integrate normpdf over
normlim = (-50.0, 50.0)

# PDF of the Cauchy distribution
cauchypdf(x) = (pi * (1.0 + x^2.0))^-1.0

# PDF of the t-distribution
function tpdf(x, dof)
	term1 = gamma(.5 * (dof + 1.0)) / (sqrt(dof * pi) * gamma(dof / 2.0))
	term2 = (1.0 + (x^2.0)/dof)^(-.5*(dof+1.0))
	return term1 * term2
end

# Generate a single random t variable
function rn_t(dof)
	z = rn_standard_normal()
	v = rn_chi_sq(dof)
	return z / sqrt(v / dof)
end

# Utility function to print each entry in an array on its own line
mapln(x) = map(println, x)

# Utility function to truncate
function truncate(x::Float64, minvalue::Float64, maxvalue::Float64)
	if x > maxvalue
		return maxvalue
	elseif x < minvalue
		return minvalue
	else
		return x
	end
end

# Truncated Cauchy
rn_trunc_cauchy(a, b) = truncate(rn_standard_cauchy(), a, b)
rn_trunc_cauchy(a, b, n) = [rn_trunc_cauchy(a, b) for x in 1:n]

# Truncated t
rn_trunc_t(dof, a, b) = truncate(rn_t(dof), a, b)
rn_trunc_t(dof, a, b, n) = [rn_trunc_t(dof, a, b) for x in 1:n]

# Extended trapezoidal rule
function exttrap(f, a, b; nmax=20, epsilon=1e-6)
	# Initialize I
	n = 2
	I = [(b - a) * .5 * ( f(a) + f(b))]

	while n < nmax
		# Determine new points to evaluate at
		irange = 1:2^(n-2)
		evalpoints = map(x -> (a + (2*x - 1) * (b - a) / (2^(n -1))), irange)

		# Sum function evals at new points
		sumterm = sum(map(f, evalpoints))

		# Add new I onto results vector
		push!(I, .5 * I[n-1] + (sumterm * (b - a) / (2^(n-1))))

		# Check for convergence
		if abs(I[n] - I[n-1]) <= epsilon * abs(I[n-1])
			return I
		end

		n += 1
	end

	return I
end

# Romberg integration
function romberg(f, a, b; nmax=20, epsilon=1e-6)
	# Get the results of the trapezoidal rule
	traps = exttrap(f, a, b, nmax=nmax, epsilon=epsilon)

	# Find a size for our results
	resultsize = min(nmax, length(traps))

	# Create a zero matrix and assign traps to the first column
	r = zeros(Float64, (resultsize, resultsize))
	r[:, 1] = traps[1:resultsize]

	# For each row past the first
	for i in 2:resultsize
		# For each column past the first
		for j in 2:i
			r[i,j] = (1 / (4^(j - 1) - 1)) * (4^(j-1) * r[i,j-1] - r[i-1,j-1])
		end

		# Check stopping criteria
		if abs(r[i,i] - r[i,i-1]) <= epsilon * abs(r[i,i-1])
			return r[1:i,1:i]
		end
	end
end

# Extended midpoint rule
function midpoint(f, a, b; nmax=20, epsilon=1e-6)
	# Initialize I
	n = 2
	I = [f(.5 * (a + b))]

	while n < nmax
		# Determine new points to evaluate at:
		# Keep odd numbers from 1:2*3^(n-1) that aren't divisible by 3
		newodds = filter(x -> x % 3 != 0, [1:2:2*3^(n-1)])

		# Divide by (2*3^(n-1)) and scale to (a,b)
		newpoints = a + (b - a) * newodds / (2*3^(n - 1))

		# Sum function evals at this point
		sumterm = sum(map(f, newpoints))

		# Add new I onto results vector
		push!(I, I[n-1] / 3.0 + sumterm * (b - a) / 3.0^(n-1))

		# Check for convergence
		if abs(I[n] - I[n-1]) <= epsilon * abs(I[n-1])
			return I
		end

		n += 1
	end

	return I
end

# Monte Carlo Integration with antithetic variables
function mci(f, a, b; nmax=1e4, epsilon=1e-6)

	# Store our random variables in a vector
	x = [rn_unif(a, b)]

	# Calculate our initial result
	I = [(b - a) * .5 * (f(x[1]) + f(a + b - x[1]))]

	# Initialize n
	n = 2
	while n < nmax
		# Generate a new random variable and add it to the vector
		push!(x, rn_unif(a, b))

		# Calculate new result and push it onto the results vector
		push!(I, (b - a) * .5 * mean(map(f, x) + map(f, a + b - x)))

		# Check for convergence
		if abs(I[n] - I[n-1]) <= epsilon * abs(I[n-1])
			return I
		end

		n += 1
	end

	return I
end

# Monte Carlo Integration with Importance Sampling (Uniform)
function mciunif(f, a, b; n=1000)
	# Store our random variables in a vector
	x = rn_unif(a, b, n)

	# Calculate our estimate
	I = (b - a) * mean(map(f, x))

	return I
end

# Monte Carlo Integration with Importance Sampling (Truncated Standard Cauchy)
function mcicauchy(f, a, b; n=1000)
	# Store our random variables in a vector
	x = rn_trunc_cauchy(a, b, n)

	# Calculate our estimate
	I = mean(map(f, x) ./ map(cauchypdf, x))

	return I
end

# Monte Carlo Integration with Importance Sampling (Truncated t-distribution)
function mcit(f, a, b; n=1000)
	# Use 30 degrees of freedom - in the future this might be worth parameterizing
	dof = 30

	# Store our random variables in a vector
	x = rn_trunc_t(dof, a, b, n)

	# Calculate our estimate
	I = mean(map(f, x) ./ map(x -> tpdf(x, dof), x))

	return I
end

# Run integration function f n times and write final results to a csv file
function mcincsv(f, n, filename)
	println("Writing $n runs to $filename")
	results = [f(x) for x in 1:n]
	writecsv(filename, results)
end

end


# # Lyapunov Function Search

#md # [![](https://mybinder.org/badge_logo.svg)](@__BINDER_ROOT_URL__/generated/Lyapunov Function Search.ipynb)
#md # [![](https://img.shields.io/badge/show-nbviewer-579ACA.svg)](@__NBVIEWER_ROOT_URL__/generated/Lyapunov Function Search.ipynb)
# **Adapted from**: SOSTOOLS' SOSDEMO2 (See Section 4.2 of [SOSTOOLS User's Manual](http://sysos.eng.ox.ac.uk/sostools/sostools.pdf))

using Test #src
using DynamicPolynomials
@polyvar x[1:3]

# We define below the vector field ``\text{d}x/\text{d}t = f``

f = [-x[1]^3 - x[1] * x[3]^2,
     -x[2] - x[1]^2 * x[2],
     -x[3] - 3x[3] / (x[3]^2 + 1) + 3x[1]^2 * x[3]]

# We need to pick an SDP solver, see [here](https://jump.dev/JuMP.jl/v0.21.6/installation/#Supported-solvers) for a list of the available choices.
# We use `SOSModel` instead of `Model` to be able to use the `>=` syntax for Sum-of-Squares constraints.

using SumOfSquares
using CSDP
solver = optimizer_with_attributes(CSDP.Optimizer, MOI.Silent() => true)
model = SOSModel(solver);

# We are searching for a Lyapunov function $V(x)$ with monomials $x_1^2$, $x_2^2$ and $x_3^2$.
# We first define the monomials to be used for the Lyapunov function:

monos = x.^2

# We now define the Lyapunov function as a polynomial decision variable with these monomials:

@variable(model, V, Poly(monos))

# We need to make sure that the Lyapunov function is strictly positive.
# We can do this with a constraint $V(x) \ge \epsilon (x_1^2 + x_2^2 + x_3^2)$,
# let's pick $\epsilon = 1$:

@constraint(model, V >= sum(x.^2))

# We now compute $\text{d}V/\text{d}x \cdot f$.

using LinearAlgebra # Needed for `dot`
dVdt = dot(differentiate(V, x), f)

# The denominator is $x[3]^2 + 1$ is strictly positive so the sign of `dVdt` is the
# same as the sign of its numerator.

P = dVdt.num

# Hence, we constrain this numerator to be nonnegative:

@constraint(model, P <= 0)

# The model is ready to be optimized by the solver:

JuMP.optimize!(model)

# We verify that the solver has found a feasible solution:

JuMP.primal_status(model)
@test JuMP.primal_status(model) == MOI.FEASIBLE_POINT #src

# We can now obtain this feasible solution with:

value(V)
@test iszero(removemonomials(value(V), monos)) #src


using Pkg
Pkg.activate(@__DIR__)
Pkg.instantiate()
##------------------------  ------------------------##
using Ipopt
using LinearAlgebra, ForwardDiff, Zygote
##------------------------  ------------------------##
# Implement hs071 from IPOPT/IPOPT.jl documentation using
# analytical and automatic differentiation.

# min x1 * x4 * (x1 + x2 + x3) + x3
# st  x1 * x2 * x3 * x4 >= 25
#     x1^2 + x2^2 + x3^2 + x4^2 = 40
#     1 <= x1, x2, x3, x4 <= 5
# Start at (1,5,5,1)
# End at (1.000..., 4.743..., 3.821..., 1.379...)

# solution/starting point = x
# objective function = f(x)
# constraints = g(x)

# number of variables
n = 4

# bounds
# 1 <= x1, x2, x3, x4 <= 5
# lower bounds
x_L = [1.0, 1.0, 1.0, 1.0]
# upper bounds
x_U = [5.0, 5.0, 5.0, 5.0]

# number of constraints, dim of g(x)
m = 2

# constraints lower bounds
# x1 * x2 * x3 * x4 >= 25
# x1^2 + x2^2 + x3^2 + x4^2 = 40
g_L = [25.0, 40.0]
# constraints upper bounds
g_U = [2.0e19, 40.0]# unbounded first term

# initial point
x_0 = [1.0, 5.0, 5.0, 1.0]

##---------------------- obj/constrain functions ----------------------##
# objective function
function eval_f(x)
    return x[1] * x[4] * (x[1] + x[2] + x[3]) + x[3]
end

# constraint function
function eval_g(x, g)
    g[1] = x[1] * x[2] * x[3] * x[4]
    g[2] = x[1]^2 + x[2]^2 + x[3]^2 + x[4]^2
    return g
end

function eval_g_ad(x, g)
    g = eltype(x).(g)
    g[1] = x[1] * x[2] * x[3] * x[4]
    g[2] = x[1]^2 + x[2]^2 + x[3]^2 + x[4]^2
    return g
end

##------------------------ grad obj function ------------------------##

# grad of obj function ∇f(x)
grad_f = zeros(n)
# analytical grad of obj function
function eval_grad_f(x::Vector{Float64}, grad_f::Vector{Float64})
    grad_f[1] = x[1] * x[4] + x[4] * (x[1] + x[2] + x[3])
    grad_f[2] = x[1] * x[4]
    grad_f[3] = x[1] * x[4] + 1
    return grad_f[4] = x[1] * (x[1] + x[2] + x[3])
end

# now with automatic differentiation
∇f = x -> ForwardDiff.gradient(eval_f, x)

# check both are the same
eval_grad_f(x_0, grad_f)
@show grad_f
@show ∇f(x_0)

function eval_grad_f_ad(x::Vector{Float64}, grad_f::Vector{Float64})
    grad_f[:] = ForwardDiff.gradient(eval_f, x)[:]
end
@show eval_grad_f_ad(x_0, grad_f)

##--------------------- jac of constraints ∇g(x)^T --------------------##
println("jacobian of constraints")
g = zeros(m)
∇g = x -> ForwardDiff.jacobian(x->eval_g_ad(x, g), x)
jac_temp = ∇g(x_0)
sparsity = jac_temp .!= 0.0
I = ((i, j) for i = 1:m, j = 1:n if sparsity[i,j] != 0.0)
rows = Int32.(getindex.(I, 1)[:])
cols = Int32.(getindex.(I, 2)[:])
values = zeros(length(rows))

# analytical version
function eval_jac_g(
    x::Vector{Float64},
    rows::Vector{Int32},
    cols::Vector{Int32},
    values::Union{Nothing,Vector{Float64}},
)
    if values === nothing
    # return the structure of the jacobian,
    # Sparse Triplet Matrix Format
    # i.e. the rows and cols of the non-zero entries
    # index style=FORTRAN STYLE (1-indexed)
        # Constraint (row) 1
        rows[1] = 1
        cols[1] = 1
        rows[2] = 1
        cols[2] = 2
        rows[3] = 1
        cols[3] = 3
        rows[4] = 1
        cols[4] = 4
        # Constraint (row) 2
        rows[5] = 2
        cols[5] = 1
        rows[6] = 2
        cols[6] = 2
        rows[7] = 2
        cols[7] = 3
        rows[8] = 2
        cols[8] = 4
    else
    # return the values of the jacobian of the constraints
        # Constraint (row) 1
        values[1] = x[2] * x[3] * x[4]  # 1,1
        values[2] = x[1] * x[3] * x[4]  # 1,2
        values[3] = x[1] * x[2] * x[4]  # 1,3
        values[4] = x[1] * x[2] * x[3]  # 1,4
        # Constraint (row) 2
        values[5] = 2 * x[1]  # 2,1
        values[6] = 2 * x[2]  # 2,2
        values[7] = 2 * x[3]  # 2,3
        values[8] = 2 * x[4]  # 2,4
    end
    @show values
    return nothing
end
println("analytical")
eval_jac_g(x_0, rows, cols, nothing)
@show rows; @show cols
eval_jac_g(x_0, rows, cols, values)
@show values
display(reshape(values, m, n))

println("\nAD")

function jac_structure!(
    m, n,
    rows::AbstractVector{<:Integer},
    cols::AbstractVector{<:Integer},
  )
    I = ((i, j) for i = 1:m, j = 1:n)
    rows .= getindex.(I, 1)[:]
    cols .= getindex.(I, 2)[:]
    return rows, cols
end

function jac_structure_sparse!(
    x, m, n,
    rows::AbstractVector{<:Integer},
    cols::AbstractVector{<:Integer},
  )
    jac_temp =  ForwardDiff.jacobian(x->eval_g_ad(x, g), x)
    sparsity = jac_temp .!= 0.0
    I = ((i, j) for i = 1:m, j = 1:n if sparsity[i,j] != 0.0)
    rows .= getindex.(I, 1)[:]
    cols .= getindex.(I, 2)[:]
    return rows, cols
end

∇g = x -> ForwardDiff.jacobian(x->eval_g_ad(x, g), x)

function eval_jac_g_ad(x, rows, cols, values)

    if values === nothing
        jac_structure_sparse!(x_0, m, n, rows, cols)
        # jac_structure!(m, n, rows, cols)
    else

        values[:] = ForwardDiff.jacobian(x->eval_g_ad(x, g), x)'[:]
    end
    return nothing
end

#jac_structure!(m, n, rows, cols)
@show jac_structure_sparse!(x_0, m, n, rows, cols)
@show rows; @show cols

eval_jac_g_ad(x_0, rows, cols, nothing)
eval_jac_g_ad(x_0, rows, cols, values)

##------------------------ Hessian of Langrangian  ------------------------##
# L = f(x) + g(x)^T * λ
# ∇L = σ_f  ∇^2 f(x_k) + Σ^m_i λ_i * ∇^2 g_i(x_k)
println("Hessian of Langrangian")
values = ones(10)

function eval_h(
    x::Vector{Float64},
    rows::Vector{Int32},
    cols::Vector{Int32},
    obj_factor::Float64,
    lambda::Vector{Float64},
    values::Union{Nothing,Vector{Float64}},
)
    if values === nothing
        # Symmetric matrix, fill the lower left triangle only
        idx = 1
        for row in 1:4
            for col in 1:row
                rows[idx] = row
                cols[idx] = col
                idx += 1
            end
        end
    else
        # Again, only lower left triangle
        # Objective
        values[1] = obj_factor * (2 * x[4])  # 1,1
        values[2] = obj_factor * (x[4])  # 2,1
        values[3] = 0                      # 2,2
        values[4] = obj_factor * (x[4])  # 3,1
        values[5] = 0                      # 3,2
        values[6] = 0                      # 3,3
        values[7] = obj_factor * (2 * x[1] + x[2] + x[3])  # 4,1
        values[8] = obj_factor * (x[1])  # 4,2
        values[9] = obj_factor * (x[1])  # 4,3
        values[10] = 0                     # 4,4

        # First constraint
        values[2] += lambda[1] * (x[3] * x[4])  # 2,1
        values[4] += lambda[1] * (x[2] * x[4])  # 3,1
        values[5] += lambda[1] * (x[1] * x[4])  # 3,2
        values[7] += lambda[1] * (x[2] * x[3])  # 4,1
        values[8] += lambda[1] * (x[1] * x[3])  # 4,2
        values[9] += lambda[1] * (x[1] * x[2])  # 4,3

        # Second constraint
        values[1] += lambda[2] * 2  # 1,1
        values[3] += lambda[2] * 2  # 2,2
        values[6] += lambda[2] * 2  # 3,3
        values[10] += lambda[2] * 2  # 4,4
    end
    return
end

println("analytical")
eval_h(x_0, Int32.(ones(10)), Int32.(ones(10)), 1., ones(2), nothing)
# Hessian of constraints
eval_h(x_0, Int32.(ones(10)), Int32.(ones(10)), 0., ones(2), values)
@show values

function L(x, λ, σ)
    λ = typeof(x)(λ)
    return σ * eval_f(x) + eval_g_ad(x,g)'*λ
end

λ = ones(2)
σ = 0.0
L(x_0, λ, σ)
∇H = x -> ForwardDiff.hessian(x->L(x, λ, σ), x)
sparsity = LowerTriangular(∇H(x_0)) .!= 0.0
I = ((i, j) for i = 1:size(sparsity,1), j = 1:size(sparsity,2) if sparsity[i,j] != 0.0)
rows_hess = Int32.(getindex.(I, 1)[:])
cols_hess = Int32.(getindex.(I, 2)[:])
values_hess = zeros(length(rows_hess))

function hess_structure_sparse!(x, rows, cols)
    ∇H = x -> ForwardDiff.hessian(x->L(x, λ, 1), x)
    sparsity = LowerTriangular(∇H(x_0)) .!= 0.0
    I = ((i, j) for i = 1:size(sparsity,1), j = 1:size(sparsity,2) if sparsity[i,j] != 0.0)
    rows .= Int32.(getindex.(I, 1)[:])
    cols .= Int32.(getindex.(I, 2)[:])
    return rows, cols
end

function eval_h_ad(
    x::Vector{Float64},
    rows,
    cols,
    σ::Float64,
    λ::Vector{Float64},
    values::Union{Nothing,Vector{Float64}},
)
    if values === nothing
        hess_structure_sparse!(x, rows, cols)
    else
        hess = ForwardDiff.hessian(x->L(x, λ, σ), x)
        values[:] = hess[tril(trues(size(hess)))]
    end
end
println("automatic differentiation")
eval_h_ad(x_0, rows_hess, cols_hess, σ, λ, values_hess)
@show rows_hess; @show cols_hess
eval_h_ad(x_0, rows_hess, cols_hess, σ, λ, values_hess)
@show values_hess

##------------------------ Solve problem analytical ------------------------##

n = 4
x_L = [1.0, 1.0, 1.0, 1.0]
x_U = [5.0, 5.0, 5.0, 5.0]

m = 2
g_L = [25.0, 40.0]
g_U = [2.0e19, 40.0]

prob = Ipopt.CreateIpoptProblem(
    n,
    x_L,
    x_U,
    m,
    g_L,
    g_U,
    8,
    10,
    eval_f,
    eval_g,
    eval_grad_f,
    eval_jac_g,
    eval_h,
)

prob.x = [1.0, 5.0, 5.0, 1.0]
solvestat = Ipopt.IpoptSolve(prob)
@show prob.x
##------------------------ solve with ad ------------------------##

n = 4
x_L = [1.0, 1.0, 1.0, 1.0]
x_U = [5.0, 5.0, 5.0, 5.0]

m = 2
g_L = [25.0, 40.0]
g_U = [2.0e19, 40.0]

g = zeros(m)
∇g = x -> ForwardDiff.jacobian(x->eval_g_ad(x, g), x)
jac_temp = ∇g(x_0)
sparsity = jac_temp .!= 0.0
I = ((i, j) for i = 1:m, j = 1:n if sparsity[i,j] != 0.0)
rows = Int32.(getindex.(I, 1)[:])
cols = Int32.(getindex.(I, 2)[:])
values = zeros(length(rows))
rows = nothing
cols = nothing
values = nothing

prob = Ipopt.CreateIpoptProblem(
    n,
    x_L,
    x_U,
    m,
    g_L,
    g_U,
    8,
    10,
    eval_f,
    eval_g,
    eval_grad_f_ad,
    eval_jac_g_ad,
    eval_h_ad,
    # function h() return false end
)
# Ipopt.AddIpoptStrOption(prob, "hessian_approximation", "limited-memory")
# Ipopt.AddIpoptStrOption(prob, "check_derivatives_for_naninf", "yes")
# Ipopt.AddIpoptIntOption(prob, "file_print_level", 10)
# Ipopt.AddIpoptIntOption(prob, "output_file", "ipopt.out")
# Ipopt.OpenIpoptOutputFile(prob, "blah.txt", 5)

prob.x = [1.0, 5.0, 5.0, 1.0]
solvestat = Ipopt.IpoptSolve(prob)
@show prob.x


struct LinearKF
    xk::Vector
	xp::Vector
	P::AbstractMatrix
	K::AbstractMatrix
	S::AbstractMatrix
	y::Vector
	filter::SquareRootFilter
end

"""
	sol = linear_kalman(
		x, x₀, P, F, H, Q, R;
		B=zeros(length(x), length(x)),
		u=zeros(length(x)),
		w=zeros(length(x)),
		v=zeros(length(x)),
		sqrt_filter=CholeskySqrt(),
		)

Performs the simple Kalman filter algorithm for the estimation of a state x.

x:   Vector with one measurement (observation) of m = length(x) variables
x₀:  A priori estimation of the variables
P:   Initial predicted process covariance matrix
F:	 Transition matrix (linear state model)
H:	 Measurement matrix (linear measurement model)
Q:   Process noise covariance matrix
R:   Measurement (observable) covariance matrix

B:	 Control-input matrix (linear state model)
u:	 Control vector (linear state model)
w:	 Process noise vector, w ~ N(0,Q)
v:	 Measurement (observable) noise vector, v ~ N(0,R)

sqrt_filter: Square root filtering technique for the covariance matrix.
			 Accepted values are: CholeskyModSqrt() (default), SVDSqrt(), CholeskySqrt()

sol: LinearKalman type
    xk:     Kalman filter state estimation
    xp:     Predicted state estimate
    P:      Process covariance matrix a posteriori
    K:      Kalman gain
	S:      Innovation (or pre-fit residual) covariance
	y:      Innovation or measurement pre-fit residual
	filter: Type of SquareRootFilter selected

"""
function linear_kalman(
	x::Vector,
    x₀::Vector,
    P::AbstractMatrix,
	F::AbstractMatrix,
	H::AbstractMatrix,
	Q::AbstractMatrix,
	R::AbstractMatrix;
	B::AbstractMatrix=zeros(length(x), length(x)),
	u::Union{Vector, Real}=zeros(length(x)),
	w::Vector=zeros(length(x)),
	v::Vector=zeros(length(x)),
	sqrt_filter::SquareRootFilter=CholeskyModSqrt(),
	)

	m = length(x)

	## Predict process
	# State estimate propagation (Predicted state, a priori)
	x̂ = _linear_state_model(x₀, F, B, u, w)
	# Error covariance propagation (Predicted process covariance matrix, a priori)
	P_ = _process_covariance_priori(P, F, Q)
	# Innovation (or pre-fit residual) covariance
	PHt = P_ * H'
	S = _innovation_covariance(P_, PHt, H, R)
	# Kalman gain
	K = _kalman_gain(P_, PHt, S)

	## Update process
	# New observation (innovation process)
    y = _linear_observation_model(x, H, v)
	# Innovation or measurement pre-fit residual
	y = muladd(-H, x̂, y)
	# State estimate update (Current state, a posteriori)
	xₖ = muladd(K, y, x̂)
	# Error covariance update (Process covariance matrix, a posteriori)
	P_ = _process_covariance_posteriori(P_, H, K, m)

	return LinearKF(vec(xₖ), vec(x̂), P_, K, S, vec(y), sqrt_filter)
end

function _linear_state_model(
	x::Vector, F::AbstractMatrix, B::AbstractMatrix, u::Union{Vector, Real}, w::Vector,
	)
	return vec(F*x .+ B .* u .+ w)
end

function _linear_observation_model(x::Vector, H::AbstractMatrix, v::Vector)
	return vec(muladd(H, x, v))
end

function _process_covariance_priori(P::AbstractMatrix, F::AbstractMatrix, Q::AbstractMatrix)
	return _return_type(P, F*P*F' .+ Q)
end

function _process_covariance_posteriori(
	P::AbstractMatrix, H::AbstractMatrix, K::AbstractMatrix, m::Int64,
	)
	return _return_type(P, (eye(m) .- K*H) *P)
end

function _kalman_gain(P::AbstractMatrix, PHt::AbstractMatrix, S::AbstractMatrix)
	F = qr(S)
	return _return_type(P, F.R \ (F.Q' * PHt))
end

function _innovation_covariance(
	P::AbstractMatrix, PHt::AbstractMatrix, H::AbstractMatrix, R::AbstractMatrix,
	)
	return _return_type(P, H*PHt .+ R)
end


using Polynomials
"""
    _getpolylagrange(k::Int64, order::Int64, origin::Int64)

Function that return the k-th Lagrange Polynomial of a certain order. Coefficients are rational then the return is exact. The polynomial is equal to :
``\\prod_{i=0,\\ i \\neq k}^{order} \\frac{x - i - origin}{k - i}``

# Arguments
- `k::Int64` : number of the Polynomial, `k` must be between `0` and `order` (`0<= k <= order`).
- `order::Int64` : order of the polynomial.
- `origin::Int64` : origin of the first indice.

# Returns
- `Polynomial{Rational{BigInt}}` : the k-th Lagrange polynomial of order `order`

# Throws
- `DommaineError` : when `0 <= k <= order` is `false` or when N ∉ {BInt64, BigInt}
"""
function _getpolylagrange(k::Int, order::Int, origin::Int) where {N<:Integer}
    0 <= k <= order || throw(DomainError("the constant 0 <= k <= order is false"))
    # the computed is made with big rational
    result = Polynomials.Polynomial([big(1 // 1)])
    for l = 0:order
        if l != k
            result *= Polynomials.Polynomial([-(l + origin), 1 // 1] .// (k - l))
        end
    end
    return result
end

"""
    Lagrange{T, edge, order, N} <: AbstractInterpolation{T, edge, order}
    Lagrange(order, T::DataType=Float64; edge::EdgeType=CircEdge)

Type containing Lagrange Polynomials coefficients for Lagrange interpolation

# Type parameters
- `T` : the type of data that is interpolate
- `edge::EdgeType` : type of edge traitment
- `order::Int`: order of lagrange interpolation

# Implementation :
- `tabfct::Vector{Polynomial{T}}` : vector of all lagrange polynomial, per example the k-th Lagrange polynomial for the designed order is tabfct[k+1]

# Arguments :
- `order::Int` : the order of interpolation
- `[T::DataType=Float64]` : The type values to interpolate

# Keywords arguments :
- `edge::EdgeType=CircEdge` : type of edge traitment

"""
struct Lagrange{T,edge,order} <: AbstractInterpolation{T,edge,order}
    tabfct::Vector{Polynomial{T}}
    function Lagrange(order::Int, T::DataType = Float64; edge::EdgeType = CircEdge)
        origin = -div(order, 2)
        tabfct_rat = collect([_getpolylagrange(i, order, origin) for i = 0:order])
        new{T,edge,order}(convert.(Polynomial{T}, tabfct_rat))
    end
end

function _c(k,n)
    p = integrate(_getpolylagrange(k,n,0))
    return p(0)-p(-1)
end
struct ABcoef
    tab::Array{Rational{Int}}
    function ABcoef(ordermax::Int)
        tab = zeros(Rational{Int}, ordermax, ordermax)
        for j=1:ordermax, i=1:j
            tab[i,j] = _c(i-1,j-1)
        end
        return new(tab)
    end
end
c(st::ABcoef, k, n)=st.tab[k,n]


"""

The sum of the primes below 10 is 2 + 3 + 5 + 7 = 17.

Find the sum of all the primes below two million.
"""

function primesum(n::Int)
    isprime = trues(n)
    isprime[1] = false

    for i in 2:Int(trunc(√n))
        if isprime[i]
            for j in i^2:i:n
                isprime[j] = false
            end
        end
    end
    return sum(filter(x->isprime[x],1:n))
end

@assert primesum(10) == 17
@show primesum(2000000)


# [1](https://projecteuler.net/problem=1)

# \sum_i ia + \sum_j jb  \text{ ,where } i \neq nb , j \neq ma

@time s = sum([ i for i in filter(x -> (x%3==0) || (x%5==0), 1:999) ])
println(s)
# slow
# 233168

# equivalent to \sum_i ia + \sum_j jb - \sum_k kab
# where kab is the repeated numbers among ia and jb.

@time s = sum([ 3:3:999; 5:5:999; -(15:15:999) ])
println(s)
# fast
# 233168


using Random, Distributions, Plots; pyplot()
Random.seed!(1)

Z() = sqrt(-2*log(rand()))*cos(2*pi*rand())
xGrid = -4:0.01:4

histogram([Z() for _ in 1:10^6], bins=50,
		normed=true, label="MC estimate")
plot!(xGrid, pdf.(Normal(),xGrid),
	 c=:red, lw=4, label="PDF",
	 xlims=(-4,4), ylims=(0,0.5), xlabel="x", ylabel="f(x)")


"""
    poweriter(A,numiter)

Perform `numiter` power iterations with the matrix `A`, starting
from a random vector. Returns a vector of eigenvalue estimates
and the final eigenvector approximation.
"""
function poweriter(A,numiter)
    n = size(A,1)
    x = normalize(randn(n),Inf)
    β = zeros(numiter)
    for k in 1:numiter
        y = A*x
        m = argmax(abs.(y))
        β[k] = y[m]/x[m]
        x = y/y[m]
    end
    return β,x
end

"""
    inviter(A,s,numiter)

Perform `numiter` inverse iterations with the matrix `A` and shift
`s`, starting from a random vector. Returns a vector of
eigenvalue estimates and the final eigenvector approximation.
"""
function inviter(A,s,numiter)
    n = size(A,1)
    x = normalize(randn(n),Inf)
    β = zeros(numiter)
    fact = lu(A - s*I)
    for k in 1:numiter
        y = fact\x
        normy,m = findmax(abs.(y))
        β[k] = x[m]/y[m] + s
        x = y/y[m]
    end
    return β,x
end

"""
    arnoldi(A,u,m)

Perform the Arnoldi iteration for `A` starting with vector `u`, out
to the Krylov subspace of degree `m`. Returns the orthonormal basis
(`m`+1 columns) and the upper Hessenberg `H` of size `m`+1 by `m`.
"""
function arnoldi(A,u,m)
    n = length(u)
    Q = zeros(n,m+1)
    H = zeros(m+1,m)
    Q[:,1] = u/norm(u)
    for j in 1:m
        # Find the new direction that extends the Krylov subspace.
        v = A*Q[:,j]
        # Remove the projections onto the previous vectors.
        for i in 1:j
            H[i,j] = dot(Q[:,i],v)
            v -= H[i,j]*Q[:,i]
        end
        # Normalize and store the new basis vector.
        H[j+1,j] = norm(v)
        Q[:,j+1] = v/H[j+1,j]
    end
    return Q,H
end

"""
    gmres(A,b,m)

Do `m` iterations of GMRES for the linear system `A`*x=`b`. Returns
the final solution estimate x and a vector with the history of
residual norms. (This function is for demo only, not practical use.)
"""
function gmres(A,b,m)
    n = length(b)
    Q = zeros(n,m+1)
    Q[:,1] = b/norm(b)
    H = zeros(m+1,m)

    # Initial solution is zero.
    x = 0
    residual = [norm(b);zeros(m)]

    for j in 1:m
        # Next step of Arnoldi iteration.
        v = A*Q[:,j]
        for i in 1:j
            H[i,j] = dot(Q[:,i],v)
            v -= H[i,j]*Q[:,i]
        end
        H[j+1,j] = norm(v)
        Q[:,j+1] = v/H[j+1,j]

        # Solve the minimum residual problem.
        r = [norm(b); zeros(j)]
        z = H[1:j+1,1:j] \ r
        x = Q[:,1:j]*z
        residual[j+1] = norm( A*x - b )
    end
    return x,residual
end

### These are not part of the text proper, but help to construct
### interesting examples.

"""
    sprandsym(n,density,λ)
    sprandsym(n,density,rcond)

Construct a randomized `n` by `n` symmetric sparse matrix of approximate
density `density`. For vector `λ`, the matrix has eigenvalues as
prescribed by λ. For scalar `rcond`, the matrix has condition number
equal to 1/`rcond`.
"""
function sprandsym(n,density,rcond::Number)
    λ = [ rcond^(i/(n-1)) for i in 0:n-1 ]
    sprandsym(n,density,λ)
end

function sprandsym(n,density,λ::Vector)
    # Apply a random Jacobi rotation similarity transformation.
    function randjr(A)
        theta = 2π*rand()
        c,s = cos(theta),sin(theta)
        i = j = 0
        while i==j
          i,j = rand(1:n,2)
        end
        A[[i,j],:] = [c s;-s c]*A[[i,j],:]
        A[:,[i,j]] = A[:,[i,j]]*[c -s;s c]
        return A
    end

    targetnz = ceil(min(0.98,density)*n^2)
    A = spdiagm(λ)

    while nnz(A) < targetnz
        A = randjr(A)
    end
    return A
end

"""
    poisson(n)

Construct the finite difference Laplacian matrix for an `n` by `n`
interior lattice.
"""
function poisson(n)
    D = spdiagm(-1=>fill(-1,n-1),0=>fill(2,n),1=>fill(-1,n-1)) * (n+1)^2/pi^2
    return kron(D,I(n)) + kron(I(n),D)
end


# Example: Compartmental models in epidemiology
#=
- https://github.com/epirecipes/sir-julia/blob/master/markdown/function_map/function_map.md
- https://en.wikipedia.org/wiki/Compartmental_models_in_epidemiology#Deterministic_versus_stochastic_epidemic_models
=#
using ModelingToolkit, Test

@inline function rate_to_proportion(r,t)
    1-exp(-r*t)
end;

# Independent and dependent variables and parameters
@parameters t c nsteps δt β γ
D = Difference(t; dt=0.1)
@variables S(t) I(t) R(t) next_S(t) next_I(t) next_R(t)

infection = rate_to_proportion(β*c*I/(S+I+R),δt)*S
recovery = rate_to_proportion(γ,δt)*I

# Equations
eqs = [D(S) ~ S-infection,
       D(I) ~ I+infection-recovery,
       D(R) ~ R+recovery]

# System
@named sys = DiscreteSystem(eqs,t,[S,I,R],[c,nsteps,δt,β,γ]; controls = [β, γ])

# Problem
u0 = [S => 990.0, I => 10.0, R => 0.0]
p = [β => 0.05, c => 10.0, γ => 0.25, δt => 0.1, nsteps => 400]
tspan = (0.0,ModelingToolkit.value(substitute(nsteps,p))) # value function (from Symbolics) is used to convert a Num to Float64
prob_map = DiscreteProblem(sys,u0,tspan,p)

# Solution
using OrdinaryDiffEq
sol_map = solve(prob_map,FunctionMap());

# Direct Implementation

function sir_map!(u_diff,u,p,t)
    (S,I,R) = u
    (β,c,γ,δt) = p
    N = S+I+R
    infection = rate_to_proportion(β*c*I/N,δt)*S
    recovery = rate_to_proportion(γ,δt)*I
    @inbounds begin
        u_diff[1] = S-infection
        u_diff[2] = I+infection-recovery
        u_diff[3] = R+recovery
    end
    nothing
end;
u0 = [990.0,10.0,0.0];
p = [0.05,10.0,0.25,0.1];
prob_map = DiscreteProblem(sir_map!,u0,tspan,p);
sol_map2 = solve(prob_map,FunctionMap());

@test Array(sol_map) ≈ Array(sol_map2)

# Delayed difference equation
@parameters t
@variables x(..) y(..) z(t)
D1 = Difference(t; dt=1.5)
D2 = Difference(t; dt=2)

@test ModelingToolkit.is_delay_var(Symbolics.value(t), Symbolics.value(x(t-2)))
@test ModelingToolkit.is_delay_var(Symbolics.value(t), Symbolics.value(y(t-1)))
@test !ModelingToolkit.is_delay_var(Symbolics.value(t), Symbolics.value(z))
@test_throws ErrorException ModelingToolkit.get_delay_val(Symbolics.value(t), Symbolics.arguments(Symbolics.value(x(t+2)))[1])
@test_throws ErrorException z(t)

# Equations
eqs = [
    D1(x(t)) ~ 0.4x(t) + 0.3x(t-1.5) + 0.1x(t-3),
    D2(y(t)) ~ 0.3y(t) + 0.7y(t-2) + 0.1z,
]

# System
@named sys = DiscreteSystem(eqs,t,[x(t),x(t-1.5),x(t-3),y(t),y(t-2),z],[])

eqs2, max_delay = ModelingToolkit.linearize_eqs(sys; return_max_delay=true)

@test max_delay[Symbolics.operation(Symbolics.value(x(t)))] ≈ 3
@test max_delay[Symbolics.operation(Symbolics.value(y(t)))] ≈ 2

linearized_eqs = [
    eqs
    x(t - 3.0) ~ x(t - 1.5)
    x(t - 1.5) ~ x(t)
    y(t - 2.0) ~ y(t)
]
@test all(eqs2 .== linearized_eqs)


function circlecenter(coords1, coords2, radius)
    # two circle intersection -> https://math.stackexchange.com/a/1367732
    dist = norm(coords1-coords2)
    midpoint = (coords1+coords2)/2
    normaldist = sqrt(radius^2 - dist^2/4)
    normalvec = reverse((coords1 - coords2)) .* [1, -1]
    normalvec /= norm(normalvec)
    center = midpoint .+ sign(radius) .* normalvec .* normaldist

    return center
end

function circlesection(coords1, coords2, center, u)
    leg1 = coords1 .- center
    leg2 = coords2 .- center

    γ = angle(vec(leg1), vec(leg2))

    x(au) = leg1[1] * cos(au*γ) - leg1[2] * sin(au*γ) + center[1]
    y(au) = leg1[1] * sin(au*γ) + leg1[2] * cos(au*γ) + center[2]

    return (x.(u), y.(u))
end

function boundaryfun(boundary::StraightBoundary, vertices, inverse_direction=false)
    coords1 = vertices[boundary.vertice_start]
    coords2 = vertices[boundary.vertice_end]

    connection = coords2 - coords1

    if inverse_direction
        return coords2, u -> coords2 - connection .* u
    else
        return coords1, u -> coords1 + connection .* u
    end
end

function boundaryfun(boundary::CircularBoundary, vertices, inverse_direction=false)
    coords1 = vertices[boundary.vertice_start]
    coords2 = vertices[boundary.vertice_end]

    center = circlecenter(coords1, coords2, boundary.radius)

    leg1 = coords1 .- center
    leg2 = coords2 .- center

    γ = angle(vec(leg1), vec(leg2))

    T(u) = [cos(u*γ) -sin(u*γ); sin(u*γ) cos(u*γ)]

    if inverse_direction
        return coords2, u -> T(1-u) * leg1 + center
    else
        return coords1, u -> T(u) * leg1 + center
    end
end


# # Non-linear effects

using Unfold
using CairoMakie
using DataFrames
using Random
using Colors

# ## Generating a non-linea signal
# We start with generating
rng = MersenneTwister(2) # make repeatable
n = 20 # datapoints
evts = DataFrame(:x=>rand(rng,n))
signal = -(3*(evts.x .-0.5)).^2 .+ 0.5 .* rand(rng,n)

plot(evts.x,signal)
#
# Looks perfectly non-linear - great!
#
# # Compare linear & non-linear fit
# First we have to reshape it to fit to Unfold format, a 3d array,  1 channel x 1 timepoint x 20 datapoints
signal = reshape(signal,length(signal),1,1)
signal = permutedims(signal,[3,2,1])
size(signal)

# Next we define three different models. **linear** **3 splines** and **10 splines**
# note the different formulas, one `x` the other `spl(x,3)`
design_linear = Dict(Any=>	(@formula(0~1+x),[0]))
design_spl3   = Dict(Any=>	(@formula(0~1+spl(x,3)),[0]))
design_spl10  = Dict(Any=>	(@formula(0~1+spl(x,10)),[0])) #hide

# and we fit the parameters
uf_linear = fit(UnfoldModel,design_linear,evts,signal);
uf_spl3 = fit(UnfoldModel,design_spl3,evts,signal);
uf_spl10 = fit(UnfoldModel,design_spl10,evts,signal); #hide

# extract the fitted values using Unfold.effects

p_linear = Unfold.effects(Dict(:x => range(0,stop=1,length=100)),uf_linear);
p_spl3 = Unfold.effects(Dict(:x => range(0,stop=1,length=100)),uf_spl3);
p_spl10 = Unfold.effects(Dict(:x => range(0,stop=1,length=100)),uf_spl10);
first(p_linear,5)

# And plot them
pl = plot(evts.x,signal[1,1,:])
	lines!(p_linear.x,p_linear.yhat)
	lines!(p_spl3.x,p_spl3.yhat)
	lines!(p_spl10.x,p_spl10.yhat)
pl

# What we can clearly see here, that the linear effect (blue) underfits the data, the `spl(x,10)` overfits it, but the `spl(x,3)` fits it perfectly.

# ## Looking under the hood
# Let's have a brief look how the splines manage what they are managing.
#
# The most important bit to understand is, that we are replacing `x` by a set of coefficients `spl(x)`.
# These new coefficients each tile the range of `x` (in our case from [0-1]) in overlapping areas, each which will be fit by one coefficient.
# Because the ranges are overlapping, we get a smooth function.
#
# Maybe this becomes clear after looking at such a basisfunction:

term_spl = Unfold.formula(uf_spl10).rhs.terms[2]

# This is the spline term.
typeof(term_spl)

# a special type generated in Unfold.jl
#
# it has a field .fun which is the spline function. We can evaluate it at a point
term_spl.fun([0.2])

# each column of this 1 row matrix is a coefficient for our regression model.
lines(disallowmissing(term_spl.fun([0.2]))[1,:])

# Note: We have to use disallowmissing, because our splines return a `missing` whenever we ask it to return a value outside it's defined range, e.g.:
term_spl.fun([-0.2])

# because it never has seen any data outside and can't extrapolate!

# Okay back to our main issue: Let's plot the whole basis set
basisSet = term_spl.fun(0.:0.01:1)
basisSet = disallowmissing(basisSet[.!any(ismissing.(basisSet),dims=2)[:,1],:]) # remove missings
ax = Axis(Figure()[1,1])
[lines!(ax,basisSet[:,k]) for k in 1:size(basisSet,2)]
current_figure()

# notice how we flipped the plot around, i.e. now on the x-axis we do not plot the coefficients, but the `x`-values
# Now each line is one basis-function of the spline.
#
# Unfold returns us one coefficient per basis-function
β=coef(uf_spl10)[1,1,:]
β = Float64.(disallowmissing(β))

# but because we used an intercept, we have to do some remodelling in the basisSet
X = hcat(ones(size(basisSet,1)), basisSet[:,1:5], basisSet[:,7:end])

# now we can weight the spline by the basisfunction
weighted = (β .*X')

# Plotting them makes for a nice looking plot!
ax = Axis(Figure()[1,1])
[lines!(weighted[k,:]) for k = 1:11]
current_figure()

# and sum them up
lines(sum(weighted,dims=1)[1,:])
plot!(X*β,color="gray") #(same as matrixproduct X*β directly!)
current_figure()

# And this is how you can think about splines


# Functions for area of various geometric shapes
# Contributed By:- [Ashwani Rathee](https://github.com/ashwani-rathee)

"""
    surfarea_cube(side)

Finds surface area of a cube

# Example

```julia
surfarea_cube(1)  # returns 6
surfarea_cube(3)  # returns 54
surfarea_cube(-1) # returns DomainError
```
"""
function surfarea_cube(side)
    if side < 0
        throw(
            DomainError(
                side,
                "surfarea_cube() only works for non-negative values",
            ),
        )
    end
    return 6(side^2)
end

"""
    surfarea_sphere(side)

Finds surface area of a sphere

# Example

```julia
surfarea_sphere(5)  # returns 314.1592653589793
surfarea_sphere(1)  # returns 12.566370614359172
surfarea_sphere(-1) # returns DomainError
```
"""
function surfarea_sphere(radius)
    if radius < 0
        throw(
            DomainError(
                radius,
                "surfarea_sphere() only works for non-negative values",
            ),
        )
    end
    return 4π * (radius^2)
end

"""
    area_rectangle(length, width)

Finds area of the rectangle

# Example

```julia
area_rectangle(10,20) # returns 200
area_rectangle(-1,-2) # returns DomainError
area_rectangle(1,-2)  # returns DomainError
area_rectangle(-1,2)  # returns DomainError
```
"""
function area_rectangle(length, width)
    if length < 0 || width < 0
        throw(
            DomainError("area_rectangle() only works for non-negative values"),
        )
    end
    return length * width
end

"""
    area_square(side)

Finds area of the area_square

# Example

```julia
area_square(10) # returns 100
area_square(-1) # returns DomainError
```
"""
function area_square(side)
    if side < 0
        throw(DomainError("area_square() only works for non-negative values"))
    end
    return side^2
end

"""
    area_triangle(base, height)

Finds area of the right angled triangle with base height

# Example

```julia
area_triangle(10,10) # returns 50.0
area_triangle(-1,-2) # returns DomainError
area_triangle(1,-2)  # returns DomainError
area_triangle(-1,2)  # returns DomainError
```
"""
function area_triangle(base, height)
    if base < 0 || height < 0
        throw(DomainError("area_triangle() only accepts non-negative values"))
    end
    return (base * height) / 2
end

"""
    area_heron_triangle(side1, side2, side3)

Finds area of a triangle using heron's formula

# Example

```julia
area_heron_triangle(5,12,13) # returns 30.0
area_heron_triangle(-1,-2,1) # returns DomainError
area_heron_triangle(1,-2,1)  # returns DomainError
area_heron_triangle(-1,2,1)  # returns DomainError
```
"""
function area_heron_triangle(side1, side2, side3)
    if side1 < 0 || side2 < 0 || side3 < 0
        throw(
            DomainError(
                "area_triangle_three_sides() only accepts non-negative values",
            ),
        )
    elseif side1 + side2 < side3 ||
           side1 + side3 < side2 ||
           side2 + side3 < side1
        throw(DomainError("Given three sides do not form a triangle"))
    end
    semi_perimeter = (side1 + side2 + side3) / 2
    area = sqrt(
        semi_perimeter *
        (semi_perimeter - side1) *
        (semi_perimeter - side2) *
        (semi_perimeter - side3),
    )
    return area
end

"""
    area_parallelogram(base, height)

Finds area of the parallelogram

# Example

```julia
area_parallelogram(10,20) # returns 200
area_parallelogram(-1,-2) # returns DomainError
area_parallelogram(1,-2)  # returns DomainError
area_parallelogram(-1,2)  # returns DomainError
```
"""
function area_parallelogram(base, height)
    if base < 0 || height < 0
        throw(
            DomainError(
                "area_parallelogram() only accepts non-negative values",
            ),
        )
    end
    return base * height
end

"""
area_trapezium(base1,base2,height)

Finds area of the traπzium

# Example

```julia
area_trapezium(10, 20, 30)  # returns 450.0
area_trapezium(-1, -2, -3)  # returns DomainError
area_trapezium(-1, 2, 3)    # returns DomainError
area_trapezium(1, -2, 3)    # returns DomainError
area_trapezium(1, 2, -3)    # returns DomainError
area_trapezium(-1, -2, 3)   # returns DomainError
area_trapezium(1, -2, -3)   # returns DomainError
area_trapezium(-1, 2, -3)   # returns DomainError
```
"""
function area_trapezium(base1, base2, height)
    if base1 < 0 || base2 < 0 || height < 0
        throw(DomainError("area_trapezium() only accepts non-negative values"))
    end
    return 1 / 2 * (base1 + base2) * height
end

"""
    area_circle(radius)

Finds area of the circle

# Example

```julia
area_circle(20) # returns 1256.6370614359173
area_circle(-1) # returns DomainError
```
"""
function area_circle(radius)
    if radius < 0
        throw(DomainError("area_circle() only accepts non-negative values"))
    end
    return π * radius^2
end

"""
    area_ellipse(radius_x, radius_y)

Finds area of the ellipse

# Example

```julia
area_ellipse(10, 10) # returns 314.1592653589793
area_ellipse(10, 20) # returns 628.3185307179587
area_ellipse(1, -2) # returns DomainError
area_ellipse(-1, 2) # returns DomainError
area_ellipse(-1, -2) # returns DomainError
```
"""
function area_ellipse(radius_x, radius_y)
    if radius_x < 0 || radius_y < 0
        throw(DomainError("area_ellipse() only accepts non-negative values"))
    end
    return π * radius_x * radius_y
end

"""
    area_rhombus(diagonal_1, diagonal_2)

Finds area of the rhombus

# Example

```julia
area_rhombus(10, 20) # returns 100.0
area_rhombus(-1,-2) # returns DomainError
area_rhombus(1,-2)  # returns DomainError
area_rhombus(-1,2)  # returns DomainError
```
"""
function area_rhombus(diagonal_1, diagonal_2)
    if diagonal_1 < 0 || diagonal_2 < 0
        throw(DomainError("area_rhombus() only accepts non-negative values"))
    end
    return 1 / 2 * diagonal_1 * diagonal_2
end

"""
    trapazoidal_area(f, x_start, x_end, steps)

Approximates the area under the curve using the trapezoidal rule
Arguments:
 - f: function for the
 - x_start: starting value for x
 - x_end: ending value for x
 - steps: steps taken while integrating.
"""
function trapazoidal_area(f, x_start, x_end, steps)
    x1 = x_start
    fx1 = f(x_start)
    area = 0.0

    for _ in 1:steps
        x2 = ((x_end - x_start) / steps) + x1
        fx2 = f(x2)
        area += abs(fx2 + fx1) * (x2 - x1) / 2
        x1 = x2
        fx1 = fx2
    end

    return area
end

"""
    area_regular_polygon(sides, side_len)

Finds area of any regular Polygon

# Example

```julia
area_regular_polygon(1, 5)  # returns DomainError
area_regular_polygon(3, 5)  # returns 10.825317547305486
area_regular_polygon(7, 15)  # returns 817.6302999003576
area_regular_polygon(-1, 4) # returns DomainError
area_regular_polygon(4, -3) # returns DomainError
area_regular_polygon(-12, -4) # returns DomainError
```
"""
function area_regular_polygon(sides, side_len)
    sides < 3 && throw(DomainError("A polygon should have atleast 3 sides"))
    (sides < 0 || side_len < 0) && throw(
        DomainError(
            "area_regular_polygon() only works for non-negative values",
        ),
    )
    return sides * side_len^2 * cot(pi / sides) / 4
end

"""
    area_polygon(V)

Finds area of any Polygon given by continuous sequence of vertex coordinates
Arguments:
- coords: x,y co-ordinates of the vertices
    [Vector of Tuples / Matrix with 2 rows or 2 columns]

# Example

```julia
area_polygon([(0, 0), (100, 0), (0, 100)])  # returns 5000.0
area_polygon([0 0;100 0;100 100;0 100])  # returns 10000.0
area_polygon([(6, 4.5), (5, 4.5), (4.5, 5.5), (5, 6.5)]) # returns 1.5
area_polygon([0 0;100 0]) # returns DomainError
area_polygon([(6, 4.63), (5, 4.63)]) # returns DomainError
```
"""
function area_polygon(coords::Matrix{T}) where {T<:Real}
    r, c = size(coords)
    (~((c == 2 && r > 2) || (r == 2 && c > 2))) &&
        throw(DomainError("A polygon should have at least 3 vertices"))

    reshape(coords, (:, 2))
    if ~(coords[1, :] == coords[end, :])
        coords = vcat(coords, coords[1, :]') #to make a loop of the vertices
    end
    return @views abs(sum(coords[1:end-1, :] .* (-coords[2:end, 2:-1:1]))) / 2
end

function area_polygon(coords::Vector{Tuple{T,N}}) where {T,N<:Real}
    length(coords) < 3 &&
        throw(DomainError("A polygon should have at least 3 vertices"))
    if ~(first(coords) == last(coords))
        push!(coords, first(coords)) #to make a loop of the vertices
    end
    res = 0
    for i in 1:length(coords)-1
        res =
            res + coords[i][1] * coords[i+1][2] - coords[i+1][1] * coords[i][2]
    end
    return abs(res) / 2
end


using Distributions, Plots; pyplot()

function anovaFStat(allData)
    xBarArray = mean.(allData)
    nArray = length.(allData)
    xBarTotal = mean(vcat(allData...))
    L = length(nArray)

    ssBetween = sum( [nArray[i]*(xBarArray[i] - xBarTotal)^2 for i in 1:L] )
    ssWithin = sum([sum([(ob - xBarArray[i])^2 for ob in allData[i]])
				for i in 1:L])
    return (ssBetween/(L-1))/(ssWithin/(sum(nArray)-L))
end

case1 = [13.4, 13.4, 13.4, 13.4, 13.4]
case2 = [12.7, 11.8, 13.4, 12.7, 12.9]
stdDevs = [2, 2, 2, 2, 2]
numObs = [24, 15, 13, 23, 9]
L = length(case1)

N = 10^5

mcFstatsH0 = Array{Float64}(undef, N)
for i in 1:N
    mcFstatsH0[i] = anovaFStat([ rand(Normal(case1[j],stdDevs[j]),numObs[j])
		for j in 1:L ])
end

mcFstatsH1 = Array{Float64}(undef, N)
for i in 1:N
    mcFstatsH1[i] = anovaFStat([ rand(Normal(case2[j],stdDevs[j]),numObs[j])
		for j in 1:L ])
end

stephist(mcFstatsH0, bins=100,
	c=:blue, normed=true, label="Equal group means case")
stephist!(mcFstatsH1, bins=100,
	c=:red, normed=true, label="Unequal group means case")

dfBetween = L - 1
dfError = sum(numObs) - 1
xGrid = 0:0.01:10
plot!(xGrid, pdf.(FDist(dfBetween, dfError),xGrid),
            c=:black, label="F-statistic analytic")
critVal = quantile(FDist(dfBetween, dfError),0.95)
plot!([critVal, critVal],[0,0.8],
	c=:black, ls=:dash, label="Critical value boundary",
	xlims=(0,10), ylims=(0,0.8), xlabel="F-value", ylabel="Density")


# This file is a part of AstroLib.jl. License is MIT "Expat".
# Copyright (C) 2016 Mosè Giordano.

function _planck_wave{T<:AbstractFloat}(wavelength::T, temperature::T)
    c1  = 3.741771790075259e-16 # = 2*pi*h*c*c
    c2  = 1.43877735382772e-2   # = h*c/k
    return c1/(wavelength^5*(expm1(c2/wavelength/temperature)))
end

"""
    planck_wave(wavelength, temperature) -> black_body_flux

### Purpose ###

Calculate the flux of a black body per unit wavelength.

### Explanation ###

Return the spectral radiance of a black body per unit wavelength using [Planck's
law](https://en.wikipedia.org/wiki/Planck%27s_law)

\$\$ B_\\lambda(\\lambda, T) =\\frac{2hc^2}{\\lambda^5}\\frac{1}{e^{\\frac{hc}{\\lambda k_\\mathrm{B}T}} - 1} \$\$

### Arguments ###

* `wavelength`: wavelength at which the flux is to be calculated, in meters.
* `temperature`: the equilibrium temperature of the black body, in Kelvin.

Both arguments can be either scalar or arrays of the same length.

### Output ###

The spectral radiance of the black body, in units of W/(sr·m³).

### Example ###

Plot the spectrum of a black body in \$[0, 3]\$ µm at \$5000\$ K.  Use
[PyPlot.jl](https://github.com/stevengj/PyPlot.jl) for plotting.

``` julia
using PyPlot
wavelength=linspace(0, 3e-6, 1000);
temperature=ones(wavelength)*5000;
flux=planck_wave(wavelength, temperature);
plot(wavelength, flux)
```

### Notes ###

`planck_freq` calculates the flux of a black body per unit frequency.

Code of this function is based on IDL Astronomy User's Library.
"""
planck_wave(w::Real, t::Real) = _planck_wave(promote(float(w), float(t))...)

@vectorize_2arg Real planck_wave


#!/usr/bin/env julia

"""
# Problem 2: Even Fibonacci numbers

Each new term in the Fibonacci sequence is generated by adding the previous two
terms. By starting with 1 and 2, the first 10 terms will be:

    1, 2, 3, 5, 8, 13, 21, 34, 55, 89, ...

By considering the terms in the Fibonacci sequence whose values do not exceed
`limit`, find the sum of the even-valued terms.
"""
module Problem002

include("Sequences.jl")

"""
    solve(; limit::Integer = 4_000_000)

Returns the solution for [`Problem002`](@ref) with the given parameters.
"""
function solve(; limit::Integer = 4_000_000)
    total = 0
    for (i, n) in enumerate(Sequences.Fibonacci())
        # don't consider terms above limit
        if n > limit
            break
        end

        # sum every third (even) Fibonacci term
        if i % 3 == 2
            total += n
        end
    end
    return total
end

end


# ### Function Contour plot

#md # ![FILE_NAME.png](images/FILE_NAME.png)

## ref https://jp.mathworks.com/matlabcentral/fileexchange/35283-matlab-plot-gallery-function-contour-plot?focused=7851904&tab=example
f(x,y) = sin(3x) * cos(x+y)

x = 0:0.01:5
y = 0:0.01:5
contour(x, y, f, xlabel="x", ylabel="y", fill=true)


"""
    shoot(ϕ,xspan,g₁,g₂,init)

Shooting method to solve a two-point boundary value problem with
ODE u'' = `ϕ`(x,u,u') for x in `xspan`, left boundary condition
`g₁`(u,u')=0, and right boundary condition `g₂`(u,u')=0. The
value `init` is an initial estimate for vector [u,u'] at x=a.

Returns vectors for the nodes, the solution u, and derivative u'.
"""
function shoot(ϕ,xspan,g₁,g₂,init,tol=1e-5)
    # ODE posed as a first-order equation in 2 variables.
    shootivp = (v,p,x) -> [ v[2]; ϕ(x,v[1],v[2]) ]

    # Evaluate the difference between computed and target values at x=b.
    function objective(s)
        IVP = ODEProblem(shootivp,s,float.(xspan))
        sol = solve(IVP,Tsit5(),abstol=tol/10,reltol=tol/10)
        x = sol.t;  y = sol;
        return [g₁(s...),g₂(y[end]...)]
    end

    # Find the unknown quantity at x=a by rootfinding.
    x = [];  y = [];   # these values will be overwritten
    s = levenberg(objective,init,xtol=tol)[:,end]

    # Use the stored last solution of the IVP.
    u,du_dx = eachrow(y)
    return x,u,du_dx
end

"""
    diffmat2(n,xspan)

Compute 2nd-order-accurate differentiation matrices on `n`+1 points
in the interval `xspan`. Returns a vector of nodes and the matrices
for the first and second derivatives.
"""
function diffmat2(n,xspan)
    a,b = xspan
    h = (b-a)/n
    x = [ a + i*h for i in 0:n ]   # nodes

    # Define most of Dₓ by its diagonals.
    dp = fill(0.5/h,n)        # superdiagonal
    dm = fill(-0.5/h,n)       # subdiagonal
    Dₓ = diagm(-1=>dm,1=>dp)

    # Fix first and last rows.
    Dₓ[1,1:3] = [-1.5,2,-0.5]/h
    Dₓ[n+1,n-1:n+1] = [0.5,-2,1.5]/h

    # Define most of Dₓₓ by its diagonals.
    d0 =  fill(-2/h^2,n+1)    # main diagonal
    dp =  ones(n)/h^2         # super- and subdiagonal
    Dₓₓ = diagm(-1=>dp,0=>d0,1=>dp)

    # Fix first and last rows.
    Dₓₓ[1,1:4] = [2,-5,4,-1]/h^2
    Dₓₓ[n+1,n-2:n+1] = [-1,4,-5,2]/h^2

    return x,Dₓ,Dₓₓ
end

"""
    diffcheb(n,xspan)

Compute Chebyshev differentiation matrices on `n`+1 points in the
interval `xspan`. Returns a vector of nodes and the matrices for the
first and second derivatives.
"""
function diffcheb(n,xspan)
    x = [ -cos( k*π/n ) for k in 0:n ]    # nodes in [-1,1]

    # Off-diagonal entries.
    c = [2; ones(n-1); 2];    # endpoint factors
    dij = (i,j) -> (-1)^(i+j)*c[i+1]/(c[j+1]*(x[i+1]-x[j+1]))
    Dₓ = [ dij(i,j) for i in 0:n, j in 0:n ]

    # Diagonal entries.
    Dₓ[isinf.(Dₓ)] .= 0         # fix divisions by zero on diagonal
    s = sum(Dₓ,dims=2)
    Dₓ -= diagm(s[:,1])         # "negative sum trick"

    # Transplant to [a,b].
    a,b = xspan
    x = @. a + (b-a)*(x+1)/2
    Dₓ = 2*Dₓ/(b-a)             # chain rule

    # Second derivative.
    Dₓₓ = Dₓ^2
    return x,Dₓ,Dₓₓ
end

"""
    bvplin(p,q,r,xspan,lval,rval,n)

Use finite differences to solve a linear bopundary value problem.
The ODE is u''+`p`(x)u'+`q`(x)u = `r`(x) on the interval `xspan`,
with endpoint function values given as `lval` and `rval`. There will
be `n`+1 equally spaced nodes, including the endpoints.

Returns vectors of the nodes and the solution values.
"""
function bvplin(p,q,r,xspan,lval,rval,n)
    x,Dₓ,Dₓₓ = diffmat2(n,xspan)

    P = diagm(p.(x))
    Q = diagm(q.(x))
    L = Dₓₓ + P*Dₓ + Q     # ODE expressed at the nodes

    # Replace first and last rows using boundary conditions.
    z = zeros(1,n)
    A = [ [1 z]; L[2:n,:]; [z 1] ]
    b = [ lval; r.(x[2:n]); rval ]

    # Solve the system.
    u = A\b
    return x,u
end

"""
    bvp(ϕ,xspan,lval,lder,rval,rder,init)

Finite differences to solve a two-point boundary value problem with
ODE u'' = `ϕ`(x,u,u') for x in `xspan`, left boundary condition
`g₁`(u,u')=0, and right boundary condition `g₂`(u,u')=0. The value
`init` is an initial estimate for the values of the solution u at
equally spaced values of x, which also sets the number of nodes.

Returns vectors for the nodes and the values of u.
"""
function bvp(ϕ,xspan,g₁,g₂,init)
    n = length(init) - 1
    x,Dₓ,Dₓₓ = diffmat2(n,xspan)
    h = x[2]-x[1]

    function residual(u)
        # Residual of the ODE at the nodes.
        du_dx = Dₓ*u                   # discrete u'
        d2u_dx2 = Dₓₓ*u                # discrete u''
        f = d2u_dx2 - ϕ.(x,u,du_dx)

        # Replace first and last values by boundary conditions.
        f[1] = g₁(u[1],du_dx[1])/h
        f[n+1] = g₂(u[n+1],du_dx[n+1])/h
        return f
    end

    u = levenberg(residual,init)
    return x,u[end]
end

"""
    fem(c,s,f,a,b,n)

Use a piecewise linear finite element method to solve a two-point
boundary value problem. The ODE is (`c`(x)u')' + `s`(x)u = `f`(x) on
the interval [`a`,`b`], and the boundary values are zero. The
discretization uses `n` equal subintervals.

Return vectors for the nodes and the values of u.
"""
function fem(c,s,f,a,b,n)
    # Define the grid.
    h = (b-a)/n
    x = @. a + h*(0:n)

    # Templates for the subinterval matrix and vector contributions.
    Ke = [1 -1; -1 1]
    Me = (1/6)*[2 1; 1 2]
    fe = (1/2)*[1; 1]

    # Evaluate coefficent functions and find average values.
    cval = c.(x);   cbar = (cval[1:n]+cval[2:n+1]) / 2;
    sval = s.(x);   sbar = (sval[1:n]+sval[2:n+1]) / 2;
    fval = f.(x);   fbar = (fval[1:n]+fval[2:n+1]) / 2;

    # Assemble global system, one interval at a time.
    K = zeros(n-1,n-1);  M = zeros(n-1,n-1);  f = zeros(n-1);
    K[1,1] = cbar[1]/h;  M[1,1] = sbar[1]*h/3;  f[1] = fbar[1]*h/2;
    K[n-1,n-1] = cbar[n]/h;  M[n-1,n-1] = sbar[n]*h/3;  f[n-1] = fbar[n]*h/2;
    for k in 2:n-1
        K[k-1:k,k-1:k] += (cbar[k]/h) * Ke
        M[k-1:k,k-1:k] += (sbar[k]*h) * Me
        f[k-1:k] += (fbar[k]*h) * fe
    end

    # Solve system for the interior values.
    u = (K+M) \ f
    u = [0; u; 0]      # put the boundary values into the result
    return x,u
end


using NPZ
using Distributions
using LinearAlgebra

#irit

"""
    generate_gaussian_data(N::Int64, D::Int64, K::Int64,MixtureVar::Number)

Generate `N` observations, generated from `K` `D` dimensions Gaussians, with the Gaussian means sampled from a `Normal` distribution with mean `0` and `MixtureVar` variance.

Returns `(Samples, Labels, Clusters_means, Clusters_cov)`

# Example
```julia
julia> x,y,clusters = generate_gaussian_data(10000,2,6,100.0)
[3644, 2880, 119, 154, 33, 3170]
...
```
"""
function generate_gaussian_data(N::Int64, D::Int64, K::Int64,MixtureVar::Number)
	x = randn(Float32,D,N)
	tpi = rand(Dirichlet(ones(K)))
	tzn = rand(Multinomial(N,tpi))
	tz = zeros(Float32,N)

	tmean = zeros(Float32,D,K)
	tcov = zeros(Float32,D,D,K)

	ind = 1
	println(tzn)
	for i=1:length(tzn)
		indices = ind:ind+tzn[i]-1
		tz[indices] .= i
		tmean[:,i] .= rand(MvNormal(zeros(Float32,D), MixtureVar*Matrix{Float32}(I, D, D)))
		tcov[:,:,i] .= rand(InverseWishart(D+2, Matrix{Float32}(I, D, D)))
		#println(minimum(eigvals(tcov[:,:,i]))>0)
		d = MvNormal(tmean[:,i], tcov[:,:,i])
		for j=indices
			x[:,j] = rand(d)
		end
		ind += tzn[i]
	end
	return x, tz, tmean, tcov
end

"""
     generate_mnmm_data(N::Int64, D::Int64, K::Int64, trials::Int64)

Generate `N` observations, generated from `K` `D` features Multinomial vectors, with `trials` draws from each vector.

Returns `(Samples, Labels, Vectors)`

# Example
```julia
julia> generate_mnmm_data(10000, 10, 5, 100)
...
```
"""
function generate_mnmm_data(N::Int64, D::Int64, K::Int64, trials::Int64)
	clusters = zeros(Float64,D,K)
	x = zeros(Float32,D,N)
	labels = rand(1:K,(N,))
	for i=1:K
		alphas = rand(1:20,(D,))
		alphas[i] = rand(30:100)
		clusters[:,i] = rand(Dirichlet(alphas))
	end
	for i=1:N
		x[:,i] = rand(Multinomial(trials,clusters[:,labels[i]]))
	end
	return x, labels, clusters
end


myParse(x) = parse.(Int, x)
crabPositions = myParse.(split.(readline("input.txt"), ','))

#! Part 1

using Statistics

function findLeastFuelPositionPt1(positions)
    return round(median(positions))
end

totalFuelCostPt1(positions, destination) = sum([abs(x-destination) for x in positions])

print(totalFuelCostPt1(crabPositions, findLeastFuelPositionPt1(crabPositions)))

#! Part 2

function findLeastFuelCostPt2(positions)
    fuel = Inf
    approxPos = Int(round(mean(positions)))
    𝛜 = 3
    for pos in approxPos-𝛜:approxPos+𝛜
        dists = [abs(x - pos) for x in positions]
        f = sum([0.5*(d^2 + d) for d in dists])

        if fuel > f
            fuel = f
        end
    end

    return fuel
end

print(findLeastFuelCostPt2(crabPositions))

# Intuition for the least fuel Position functions: https://www.johnmyleswhite.com/notebook/2013/03/22/modes-medians-and-means-an-unifying-perspective/


# p7.m - accuracy of periodic spectral differentiation

# Compute derivatives for various values of N:
Nmax = 50; allN = 6:2:Nmax; E = zeros(4,length(allN));
for N = 6:2:Nmax
    h = 2*pi/N; x = h*(1:N);
    column = [0; @. .5*(-1)^(1:N-1)*cot((1:N-1)*h/2)];
    D = toeplitz(column,column[[1;N:-1:2]]);
    v = @. abs(sin(x))^3;                     # 3rd deriv in BV
    vprime = @. 3*sin(x)*cos(x)*abs(sin(x));
    j = round(Int,N/2-2);
    E[1,j] = norm(D*v-vprime,Inf);
    v = @. exp(-sin(x/2)^(-2));               # C-infinity
    vprime = @. .5*v*sin(x)/sin(x/2)^4;
    E[2,j] = norm(D*v-vprime,Inf);
    v = @. 1/(1+sin(x/2)^2);                 # analytic in a strip
    vprime = @. -sin(x/2)*cos(x/2)*v^2;
    E[3,j] = norm(D*v-vprime,Inf);
    v = sin.(10*x); vprime = 10*cos.(10*x);   # band-limited
    E[4,j] = norm(D*v-vprime,Inf);
end

# Plot results:
titles = [L"|\sin(x)|^3",L"\exp(-\sin^{-2}(x/2))",L"1/(1+\sin^2(x/2))",L"\sin(10x)"]; clf();
for iplot = 1:4
    subplot(2,2,iplot);
    semilogy(allN,E[iplot,:],".-",markersize=6);
    axis([0,Nmax,1e-16,1e3]); grid(true);
    xticks(0:10:Nmax); yticks(10.0.^(-15:5:0));
    title(titles[iplot]);
    iplot > 2 ? xlabel("N") : nothing;
    iplot%2 > 0? ylabel("error") : nothing;
end


A = randn(3,3)

x = fill(1.0, (3))
b = A*x

l,u,p = lu(A)

norm(l*u - A)

norm(l*u - A[p,:])

l,u,p = lu(A, Val{false})

norm(l*u - A)

Alu = lufact(A)

Alu[:P]

Alu[:L]

Alu[:U]

# PA = LU
# A = P'LU
# P'LUx = b
# LUx = Pb
# Ux = L\Pb
# x = U\L\Pb
Alu[:U]\(Alu[:L]\(Alu[:P]b))

Alu\b

det(A)

det(Alu)

Atall = randn(3, 2)

Aqr = qrfact(Atall)

Aqr[:Q]

Aqr[:R]

Aqr[:Q]*ones(2)

Aqr[:Q]*ones(3)

Aqr[:Q]*ones(4)

Aqr[:Q][1]

Aqr\b

Atall\b

v = randn(3)
# Tomar el producto exterio de un vector consigo mismo nos da una matriz singular
singmatriz = v * v'

Aqrp = qrfact(singmatriz,Val{true})

Aqrp\b

Asym = A + A'
AsymEig = eigfact(Asym)

AsymEig[:values]

AsymEig[:vectors]

inv(AsymEig)*Asym

eig(Asym)

Asvd = svdfact(A[:,1:2])

Asvd\b

A

Diagonal(diag(A))

Diagonal(A)

LowerTriangular(tril(A))

LowerTriangular(A)

Symmetric(Asym)

SymTridiagonal(diag(Asym),diag(Asym,1))

n = 1000;
A = randn(n,n);

Asym = A + A';

Asym_noisy = copy(Asym); Asym_noisy[1,2] += 5eps();

println("Is Asym symmetric? ", issymmetric(Asym))
println("Is Asym_noisy symmetric? ", issymmetric(Asym_noisy))

@time eigvals(Asym);

@time eigvals(Asym_noisy);

@time eigvals(Symmetric(Asym_noisy));

n = 1_000_000;
A = SymTridiagonal(randn(n), randn(n-1));
@time eigmax(A)

rand(1:10,3,3)*rand(1:10,3,3)

1//2

Ar = convert(Matrix{Rational{BigInt}}, rand(1:10,3,3))/10

x = fill(1, (3))
b = Ar*x

Ar\b

lufact(Ar)

λ1,λ2,λ3 = 1//1,1//2,1//4
v1,v2,v3 = [1,0,0],[1,1,0],[1,1,1]
V,Λ = [v1 v2 v3], Diagonal([λ1,λ2,λ3])
A = V*Λ/V

es-es/intro-to-julia-ES/plutos/11.Factorizaciones y otras diversiones.jl


using Plots

function ackley(x, a=20, b=0.2, c=2π)
    d = length(x);
    return -a * exp(-b * sqrt(sum(x.^2)/d)) -
                exp(sum(cos(c*xi) for xi in x)/d) + a + exp(1);
end

# plot

contour(-30:0.1:30, -30:0.1:30, (x, y)->ackley([x,y]))
heatmap(-30:0.1:30, -30:0.1:30, (x, y)->ackley([x,y]))


function onedElliptic(nElements=40,order=2)
#  Solves a linear elliptic PDE in 1D

  f(x)=x           # right-hand side function definition.

  x,eConn,iU,iC = onedMesh( [0.0,1.0], order, nElements )
  nNodes = size(x,1)

  ide = [1:nNodes;]     # index into unknowns  (set b.c. later in 1D)

  r,w = onedQuadratureRule(5)
  o   = ones(size(r))

  nElDOF = order+1
  nEntries = nElements*nElDOF^2
  II  = zeros(Int64,nEntries) #Array{Int32,1}(undef,nEntries)     # zeros(Int32,nEntries)
  JJ  = zeros(Int64,nEntries) #Array{Int32,1}(undef,nEntries)     # zeros(Int32,nEntries)
  AA  = zeros(Float64,nEntries) #Array{Float64,1}(undef,nEntries)   # zeros(Float64,nEntries)
  b   = zeros(Float64,nNodes) #Array{Float64,1}(undef,nNodes)     # zeros(Float64,nNodes)

  index = 0
  for k=1:nElements
    nLocal = eConn[k,:]
    xLocal = view(x,nLocal)

    xg,wg,phi,p_x,p_xx = onedShape( xLocal, r, w )
    fg     = f(xg)             # forcing function evaluated at quadrature points

    ALoc   = onedBilinear(  o, p_x, p_x, wg )
    bLoc   = onedLinForm(  fg,      phi, wg )

    lDOF = ide[nLocal]
    for nt = 1:nElDOF
      for nu = 1:nElDOF
        index = index + 1
        II[index]   = lDOF[nu]
        JJ[index]   = lDOF[nt]
        AA[index]   = ALoc[nt,nu]
      end
      b[lDOF[nt]] = b[lDOF[nt]] + bLoc[nt]
    end
  end

  A = sparse(II,JJ,AA)

  u = A[iU,iU]\b[iU]

  #  Calculate the exact solution for unit test
  xi = x[iU];

  uExact = -(xi.^3-xi)/6

  return norm(u-uExact)
# using Gaston
# figure(1)
# Gaston.set_terminal("x11");
# Gaston.closeall();
# c = Gaston.CurveConf();
# c.color = "blue";
# c.legend = "FEM";
# Gaston.addcoords(xi,u,c);

# figure(2)
# Gaston.set_terminal("x11");
# c = Gaston.CurveConf();
# c.color = "red";
# c.legend = "Exact";
# c.marker = "fdmd";
# c.plotstyle = "linespoints";
# Gaston.addcoords(xi,-(xi.^3-xi)/6,c);
# Gaston.llplot();

# set_filename("test.png")
# printfigure("png")

#  using PyPlot
#  plot(xi,u,"r",label="FEM solution")
#  plot(xi,-(xi.^3-xi)/6,"go",label="Exact at nodes")
#  legend()
end


"""
    panas(f,c1,c2,a,b,c,m,n)
berisi program untuk mencari solusi
persamaan differensial parsial yaitu persamaan panas. Program ini secara default berisi
8 masukan, yaitu fungsi nilai awal `f(x)`, koefisien nilai batas `c1` dan `c2`, nilai batas solusi
`a` dan `b`, koefisien persamaan gelombang `c`, dan jumlah grid solusi x dan t yaitu `m` dan
`n`.
# Example
```jl
julia> f(x) = 4*x - 4*x.^2;

julia> U,R = panas(f,0,0,1,0.2,1,6,11);

julia> R
0.4999999999999999

julia> U
11×6 Adjoint{Float64,Array{Float64,2}}:
 0.0  0.64       0.96      0.96      0.64       0.0
 0.0  0.48       0.8       0.8       0.48       0.0
 0.0  0.4        0.64      0.64      0.4        0.0
 0.0  0.32       0.52      0.52      0.32       0.0
 ⋮                                              ⋮
 0.0  0.1375     0.2225    0.2225    0.1375     0.0
 0.0  0.11125    0.18      0.18      0.11125    0.0
 0.0  0.09       0.145625  0.145625  0.09       0.0
 0.0  0.0728125  0.117813  0.117813  0.0728125  0.0
```
return solusi persamaan panas `U` dan rasio `R`.
"""
function panas(f,c1,c2,a,b,c,m,n)
  h = a/(m-1);
  k = b/(n-1);
  r = c^2*k/h^2;
  U = zeros(m,n);
  U[1,:] .= c1;
  U[m,:] .= c2;
  U[2:m-1,1] = f.(h:h:(m-2)*h)';
  for j = 2:n
    for i = 2:m-1
      U[i,j]=(1-2*r)*U[i,j-1]+ r*(U[i-1,j-1]+U[i+1,j-1]);
    end
  end
  U=U';
  return U,r
end


"""
   Implementing the ideas in

      @ARTICLE{1993JCoPh.104..211D,
          author = {{Dang-Vu}, H. and {Delcarte}, C.},
           title = "{An Accurate Solution of the Poisson Equation by the Chebyshev Collocation Method}",
         journal = {Journal of Computational Physics},
            year = 1993,
           month = jan,
          volume = {104},
          number = {1},
           pages = {211-220},
             doi = {10.1006/jcph.1993.1021},
          adsurl = {https://ui.adsabs.harvard.edu/abs/1993JCoPh.104..211D},
         adsnote = {Provided by the SAO/NASA Astrophysics Data System}
      }
"""
module ChebyshevSpec

include("CustomTypes.jl")
using .CustomTypes
using SparseArrays

export mat_invD1, mat_invD2

function mat_invD1(nx::myI)
   X = Vector{myF}()
   Y = Vector{myF}()
   V = Vector{myF}()

   c(n) = (n==0)      ? 2.0 : 1.0
   e(n) = (n<=(nx-1)) ? 1.0 : 0.0

   for i=2:(nx-1)
      n=i-1
      push!(X,i)
      push!(Y,i-1)
      push!(V,tomyF(c(n-1)*0.5/n))

      push!(X,i)
      push!(Y,i+1)
      push!(V,tomyF(-e(n+1)*0.5/n))
   end

   return dropzeros(sparse(X,Y,V))
end

function mat_invD2(nx::myI)
   X = Vector{myF}()
   Y = Vector{myF}()
   V = Vector{myF}()

   c(n) = (n==0)      ? 2.0 : 1.0
   e(n) = (n<=(nx-1)) ? 1.0 : 0.0

   for i=3:(nx-2)
      n=i-1
      push!(X,i)
      push!(Y,i-2)
      push!(V,tomyF(c(n-2)*0.25/(n*(n-1))))

      push!(X,i)
      push!(Y,i)
      push!(V,tomyF(-e(n+2)*0.5/((n+1)*(n-1))))

      push!(X,i)
      push!(Y,i+2)
      push!(V,tomyF(e(n+4)*0.25/(n*(n+1))))
   end
   n=nx-1
   push!(X,nx-1)
   push!(Y,nx-3)
   push!(V,tomyF(c(n-2)*0.25/(n*(n-1))))

   n=nx
   push!(X,nx)
   push!(Y,nx-2)
   push!(V,tomyF(c(n-2)*0.25/(n*(n-1))))

   return dropzeros(sparse(X,Y,V))
end

end


function KL(p::AbstractMvNormal, q::AbstractMvNormal)
    # This is the generic implementation for AbstractMvNormal, you might need to specialize for your type
    length(p) == length(q) ||
        throw(DimensionMismatch("Distributions p and q have different dimensions $(length(p)) and $(length(q))"))
    Σp, Σq = cov.((p, q))
    Δμ = mean(p) - mean(q)
    0.5 * (tr(Σq \ Σp) + dot(Δμ, Σq \ Δμ) - length(p) + logdet(Σq) - logdet(Σp))
end

function KL(p::MvNormal, q::MvNormal)
    # We use p.Σ and q.Σ to take the advantage that they are defined as PDMats objects
    length(p) == length(q) ||
        throw(DimensionMismatch("Distributions p and q have different dimensions $(length(p)) and $(length(q))"))
    0.5 * (tr(q.Σ \ p.Σ) + invquad(q.Σ, mean(p) - mean(q)) - length(p) + logdet(q.Σ) - logdet(p.Σ))
end


function demoChainIndepRational()

    println("In this demo we consider the directed graph A->B->C")
    println("The chain is such that A and B are dependent, B and C are dependent, yet A and C are independent.")
    println("To show this we will define the table entries using rationals. Julia will then use rational arithmetic for the subsequent calculations.")

    A,B,C=1,2,3
    pA=PotArray(A,[3//5, 2//5])
    pBgA=PotArray([B A],[1//4 15//40; 1//12 1//8; 2//3 1//2])
    pCgB=PotArray([C B],[1//3 1//2 15//40; 2//3 1//2 5//8])
    pABC=pCgB*pBgA*pA
    pAC=sum(pABC,B)
    pA=sum(pAC,C)
    pC=sum(pAC,A)

    println("pAC-pA*pC=")
    println(pAC-pA*pC)
    println("This shows that A and C are independent.\n")

    println("Contrast this with the same calculation using floating point arithmetic:")

    pA=PotArray(A,[3/5, 2/5])
    pBgA=PotArray([B A],[1/4 15/40; 1/12 1/8; 2/3 1/2])
    pCgB=PotArray([C B],[1/3 1/2 15/40; 2/3 1/2 5/8])
    pABC=pCgB*pBgA*pA
    pAC=sum(pABC,B)
    pA=sum(pAC,C)
    pC=sum(pAC,A)

    println("pAC-pA*pC=")
    println(pAC-pA*pC)
    println("The loss of numerical accuracy means that we cannot confidently declare A and C are independent, even though they actually are.")

end


### A Pluto.jl notebook ###
# v0.11.14

using Markdown
using InteractiveUtils

# This Pluto notebook uses @bind for interactivity. When running this notebook outside of Pluto, the following 'mock version' of @bind gives bound variables a default value (instead of an error).
macro bind(def, element)
    quote
        local el = $(esc(element))
        global $(esc(def)) = Core.applicable(Base.get, el) ? Base.get(el) : missing
        el
    end
end

# ╔═╡ 0dcfd858-f867-11ea-301c-c3ca0a224117
using Plots

# ╔═╡ 1e058ba2-ec94-11ea-09af-7f9f9cc3a233
using PlutoUI

# ╔═╡ 13f6ccac-7ce0-48d7-a0ef-e83489625e1d
using Statistics

# ╔═╡ 78763674-f8fe-11ea-349c-d997f30ac1f6
using LinearAlgebra

# ╔═╡ 35e83a04-f864-11ea-0a8e-9ddf6eec02f3
using Images, ColorSchemes

# ╔═╡ 16887070-f891-11ea-2db3-47b91930e728
using ForwardDiff

# ╔═╡ 3b9941ac-6043-4dc6-850f-4c7b3ae9d9a7
md"""
# Finding structure in data
"""

# ╔═╡ 7365084a-1f37-4897-bca4-fc5855c5ee4e
md"""
In this notebook we will look at one way to analyse, understand and simplify data. We will look at the ideas and intuition behind **principal component analysis**, a technique to find the most important directions and dimensions in a data set. This is very closely related to the SVD (singular-value decomposition) that we saw on Tuesday, and is one way to find and exploit structure in data.
"""

# ╔═╡ ed7ff6b2-f863-11ea-1a59-eb242a8674e3
md"## Flags"

# ╔═╡ fed5845e-f863-11ea-2f95-c331d3c62647
md"Let's start off by recalling the idea of a **multiplication table**, also called an **outer product**:"

# ╔═╡ 0e1a6d80-f864-11ea-074a-5f7890180114
outer(v, w) = [x * y for x in v, y in w]

# ╔═╡ 2e497e30-f895-11ea-09f1-d7f2c1f61193
outer(1:10, 1:10)

# ╔═╡ ab3d55cc-f905-11ea-2f22-5398f3aca803
md"Some flags are a simple example of this:"

# ╔═╡ 13b6c108-f864-11ea-2447-2b0741f15c7b
flag = outer([1, 0.1, 2], ones(6))

# ╔═╡ e66b30a6-f914-11ea-2c0f-35282d45a30a
ones(6)

# ╔═╡ 71d1b12e-f895-11ea-39df-f5c18a7766c3
flag2 = outer([1, 0.1, 2], [1, 1, 1, 3, 3, 3])

# ╔═╡ cdbe1d8e-f905-11ea-3884-efeeef386dda
md"## Rank of a matrix"

# ╔═╡ d9aa9af0-f865-11ea-379e-f16b452bd94c
md"A matrix that can be written exactly as a single multiplication table is called a **rank-1** matrix. If it can be written as the *sum* of *two* multiplication tables, it is called **rank 2**, etc."

# ╔═╡ 2e8ae92a-f867-11ea-0219-1bdd9627c1ea
md"Let's see what a general rank-1 matrix looks like:"

# ╔═╡ 38adc490-f867-11ea-1de5-3b633aff7c97
image = outer([1; 0.4; rand(50)], rand(500));

# ╔═╡ 9cf23f9a-f864-11ea-3a08-af448aceefd8
md"Now what happens if we add a little bit of **noise**, i.e. randomness?"

# ╔═╡ a5b62530-f864-11ea-21e8-71ccfed487f8
noisy_image = image .+ 0.03 .* randn.();

# ╔═╡ c41df86c-f865-11ea-1253-4942bbdbe9d2
md"""The noisy image now has a rank larger than 1. But visually we can see that it is "close to" the original rank-1 matrix.

Given this matrix, how can we discover that it is close to a structured, rank-1 matrix? We would like to be able to find this out and say that the matrix is close to a simple one."""

# ╔═╡ 7fca33ac-f864-11ea-2a8b-933eb382c172
md"## Images as data"

# ╔═╡ 283f5da4-f866-11ea-27d4-957ca2551b92
md"Images are just one example of the many different types of data we come across in the world.

Let's treat the image as a **data matrix**, where each column of the image / matrix is a **vector** representing one observation of data. (In data science it is often the rows that correspond to observations.)

Let's try to visualize those vectors, taking just the first two rows of the image as the $x$ and $y$ coordinates of our data points:"

# ╔═╡ 54977286-f908-11ea-166d-d1df33f38454
image[1:2, 1:20]

# ╔═╡ 7b4e90b4-f866-11ea-26b3-95efde6c650b
begin
	xx = image[1, :]
	yy = image[2, :]
end

# ╔═╡ f574ad7c-f866-11ea-0efa-d9d0602aa63b
md"## Plotting data"

# ╔═╡ 8775b3fe-f866-11ea-3e6f-9732e39a3525
md"We would like to **visualise** this data. There are various plotting packages that we could use. We will use `Plots.jl`:"

# ╔═╡ 7bacf44e-f896-11ea-38be-2b16ae7ca99f
scatter(xx, yy, alpha=0.5, framestyle=:origin, label="original image", leg=:topleft,
		xlabel="x values", ylabel="y values")

# ╔═╡ 1147cbda-f867-11ea-08fa-ef6ed2ae1e93
begin
	scatter(xx, yy,
			leg=:topleft, label="rank-1", ms=3, alpha=0.3,
			size=(500, 400), m=:square, c=:red,
			framestyle=:origin)

	xs = noisy_image[1, :]
	ys = noisy_image[2, :]

	scatter!(xs, ys, label="noisy", m=:., alpha=0.3, c=:blue)
end

# ╔═╡ 8a611e36-f867-11ea-121f-317b7c145fe3
md"We see that the exact rank-1 matrix has columns that **lie along a line**, since they are just multiples of one another.

The approximate rank-1 matrix has columns that **lie *close to* a line**!"

# ╔═╡ f7371934-f867-11ea-3b53-d1566684585c
md"So, given the data, we want to look at it do see if it lies close to a line or not.
How can we do so in an *automatic* way?
"

# ╔═╡ 119dc35c-ec94-11ea-190c-23a750fbe7f4
md"The data are given by pairs $(x_i, y_i)$. We can highlight the $i$th data point in the set to see how the data are spread out. Remember that this is literally scanning through the columns of the image!"

# ╔═╡ 2043d4e6-ec94-11ea-1e1a-c75742eafe71
@bind i Slider(1:length(xs), show_value=true)

# ╔═╡ 2a705962-ec94-11ea-1181-2f001ccf472f
begin
	scatter(xs, ys, ms=4, alpha=0.5, ratio=1, leg=false, size=(500, 400),
			framestyle=:origin)
	scatter!([xs[i]], [ys[i]], ms=8, alpha=0.8, c=:red)
end

# ╔═╡ 987c1f2e-f868-11ea-1125-0d8c02843ae4
md"## Size of the data cloud"

# ╔═╡ 9e78b048-f868-11ea-192e-d903265d1eb5
md"Looking at this cloud of data points, a natural thing to do is to try to *measure* it: How wide is it, and how tall?"

# ╔═╡ 24df1f32-ec90-11ea-1f6d-03c1bfa5df8e
md"""For example, let's think about calculating the width of the cloud, i.e. the range of possible $x$-values of the data. For this, the $y$-values are actually irrelevant.

A first step in analysing data is often to **centre** it data around 0 by subtracting the mean. This is sometimes called "de-meaning":
"""

# ╔═╡ aec46a9b-f743-4cbd-97a7-3ef3cac78b12
begin
	xs_centered = xs .- mean(xs)
	ys_centered = ys .- mean(ys)
end

# ╔═╡ 1b8c743e-ec90-11ea-10aa-e3b94f768f82
scatter(xs_centered, ys_centered, ms=5, alpha=0.5, ratio=1, leg=false, framestyle=:origin)

# ╔═╡ f5358ce4-f86a-11ea-2989-b1f37be89183
md"A common way (but not the only way!) to calculate the width of a data set is the **standard deviation**, i.e. the square distance of the centered data from the origin. We will do this *separately* for $x$ and $y$ by *projecting* onto that direction -- i.e. effectively ignoring the other coordinate:"

# ╔═╡ 870d3efa-f8fc-11ea-1593-1552511dcf86
begin
	scatter(xs_centered, ys_centered, ms=5, alpha=0.5, ratio=1, leg=false, framestyle=:origin)

	scatter!(xs_centered, zeros(size(xs_centered)), ms=5, alpha=0.1, ratio=1, leg=false, framestyle=:origin)

	for i in 1:length(xs_centered)
		plot!([(xs_centered[i], ys_centered[i]), (xs_centered[i], 0)], ls=:dash, c=:black, alpha=0.1)
	end

	plot!()

end

# ╔═╡ 03ab44c0-f8fd-11ea-2243-1f3580f98a65
md"This gives the following approximate extents (standard deviations) of the cloud:"

# ╔═╡ 2c3721da-f86b-11ea-36cf-3fe4c6622dc6
begin
	width = sqrt(mean(xs_centered.^2))
	height = sqrt(mean(ys_centered.^2))
end

# ╔═╡ 6dec0db8-ec93-11ea-24ad-e17870ee64c2
begin
	scatter(xs_centered, ys_centered, ms=5, alpha=0.5, ratio=1, leg=false,
			framestyle=:origin)

	vline!([-2*width, 2*width], ls=:dash, lw=1.5)
	hline!([-2*height, 2*height], ls=:dash, lw=1.5)
end

# ╔═╡ 5fab2c32-f86b-11ea-2f27-ed5feaac1fa5
md"We expect most (~95%) of the data to be contained within the mean $\pm$ (twice the standard deviation)."

# ╔═╡ ae9a2900-ec93-11ea-1ae5-0748221328fc
md"## Correlated data"

# ╔═╡ b81c9db2-ec93-11ea-0dbd-4bd0951cb2cc
md"""However, from the figure we see that $x$ and $y$ are not the correct directions to think about for this data set. It would be more natural to think about other directions: the direction in which the data set is mainly pointing (roughly, the direction in which it's longest) and the approximately perpendicular direction in which it is most narrow.

We need to find from the data *which* directions these are, and the width in those directions.

We cannot get that information by looking at $x$-coordinates and $y$-coordinates separately; rather, it is encoded in the *relationship* between the values of $x_i$ and $y_i$ for those points $(x_i, y_i)$ that are in the data set.

For example, when $x$ is large and negative, $y$ is also quite negative; when $x$ is 0, $y$ is near $0$, and when $x$ is large and positive, so is $y$. We say that $x$ and $y$ are **correlated** -- literally they are mutually ("co") related, such that knowing some information about one of them allows us to predict something about the other.
"""

# ╔═╡ 80722856-f86d-11ea-363d-53fc5f6b8152
md"There are standard ways of calculating this correlation, but we prefer to hone our **intuition** using computational thinking instead!"

# ╔═╡ b8fa6a1c-f86d-11ea-3d6b-2959d737254b
md"We want to think about different *directions*, so let's introduce an angle $\theta$ to describe the direction along which we are looking. We want to calculate the width of the cloud along that direction.

Effectively we are *changing coordinates* to a new coordinate oriented along the line. To do this fully requires more linear algebra than we are assuming in this course, but let's see what it looks like:"

# ╔═╡ 3547f296-f86f-11ea-1698-53d3c1a0bc30
md"## Rotating the data"

# ╔═╡ 7a83101e-f871-11ea-1d87-4946162777b5
md"""By rotating the data we can look in different directions and calculate the width of the data set "along that direction". Again, what we are really doing is **projecting** the data onto that direction."""

# ╔═╡ e8276b4e-f86f-11ea-38be-218a72452b10
M = [xs_centered ys_centered]'

# ╔═╡ d71fdaea-f86f-11ea-1a1f-45e4d50926d3
imax = argmax(M[1, :])

# ╔═╡ 757c6808-f8fe-11ea-39bb-47e4da65113a
svdvals(M)

# ╔═╡ cd9e05ee-f86f-11ea-0422-25f8329c7ef2
R(θ)= [cos(θ) sin(θ)
	  -sin(θ) cos(θ)]

# ╔═╡ 7eb51908-f906-11ea-19d2-e947d81cb743
md"In the following figure, we are rotating the axis (red arrow) around in the left panel. In the right panel we are viewing the data from the point of view of that new coordinate direction, effectively as if we rotated our head so the red vector was horizontal:"

# ╔═╡ 4f1980ea-f86f-11ea-3df2-35cca6c961f3
@bind θ Slider(0:0.01:2π, show_value=true, default=0.0)

# ╔═╡ 3b71142c-f86f-11ea-0d43-47011d00786c
p1 = begin

	scatter(M[1, :], M[2, :], ratio=1, leg=false, ms=2.5, alpha=0.5,
			framestyle=:origin)

	plot!([0.7 .* (-cos(θ), -sin(θ)), 0.7 .* (cos(θ), sin(θ))], lw=2, arrow=true, c=:red, alpha=0.8)
	xlims!(-0.7, 0.7)
	ylims!(-0.7, 0.7)

	scatter!([M[1, imax]], [M[2, imax]],  ms=5, alpha=1, c=:yellow)

	annotate!(0, 1.2, text("align arrow with cloud", :red, 10))
end;

# ╔═╡ 88bbe1bc-f86f-11ea-3b6b-29175ddbea04
p2 = begin
	M2 = R(θ) * M

	scatter(M2[1, :], M2[2, :],ratio=1, leg=false, ms=2.5, alpha=0.5, framestyle=:origin)
	# plot!([(-1, 0), (1, 0)], lw=3, arrow=true, c=:red)

	scatter!([M2[1, imax]], [M2[2, imax]], ms=5, alpha=1, c=:yellow)

	xlims!(-0.7, 0.7)
	ylims!(-0.7, 0.7)

	σ = std(M2[1, :])
	vline!([-2σ, 2σ], ls=:dash, lw=2)
end;

# ╔═╡ 2ffe7ed0-f870-11ea-06aa-390581500ca1
plot(p1, p2)

# ╔═╡ a5cdad52-f906-11ea-0486-755a6403a367
md"Let's plot the variance in a direction $\theta$ as a function of $\theta$:"

# ╔═╡ 0115c974-f871-11ea-1204-054510848849
begin
	f(θ) = var((R(θ) * M)[1,:])
	f(θ::AbstractArray) = f(θ[1])
end

# ╔═╡ 0935c870-f871-11ea-2a0b-b1b824379350
begin
	plot(0:0.01:2π, f, leg=false, size=(400, 300))

	xlabel!("θ")
	ylabel!("variance in direction θ")
end

# ╔═╡ e4af4d26-f877-11ea-1de3-a9f8d389138e
md"""The direction in which the variance is **maximised** gives the most important direction, the first **principal component**. We can quantify how close the data is to being along a single line using the width in the perpendicular direction; if it is "very small" compared to the width in the first principal direction then the data is close to being rank 1."""

# ╔═╡ bf57f674-f906-11ea-08eb-9b50818a025b
md"The simplest way to maximise this function is to evaluate it everywhere and find one of the places where it takes the maximum value:"

# ╔═╡ 17e015fe-f8ff-11ea-17b4-a3aa072cd7b3
begin
	θs = 0:0.01:2π
	fs = f.(θs)

	θmax = θs[argmax(fs)]
	θmin = θs[argmin(fs)]

	fmax = f(θmax)
	fmin = f(θmin)
end

# ╔═╡ 045b9b98-f8ff-11ea-0d49-5b209319e951
begin
	scatter(xs_centered, ys_centered, ms=5, alpha=0.3, ratio=1, leg=false,
			framestyle=:origin)

	plot!([(0, 0), 2*sqrt(fmax) .* (cos(θmax), sin(θmax))], arrow=true, lw=3, c=:red)
	plot!([(0, 0), 2*sqrt(fmin) .* (cos(θmin), sin(θmin))], arrow=true, lw=3, c=:red)

end

# ╔═╡ cfec1ec4-f8ff-11ea-265d-ab4844f0f739
md"Note that the directions that maximise and minimise variance are perpendicular. This is always the case.

We can think of this procedure as effectively *fitting an ellipse* to the data. The widths of the ellipse axes show the relative importance of each direction in the data."

# ╔═╡ e6e900b8-f904-11ea-2a0d-953b99785553
begin
	circle = [cos.(θs) sin.(θs)]'
	stretch = [2 * sqrt(fmax) 0
				0   		2 * sqrt(fmin)]
	ellipse = R(-θmax) * stretch * circle

	plot!(ellipse[1, :], ellipse[2, :], series=:shape, alpha=0.4, fill=true, c=:orange)
end

# ╔═╡ aaff88e8-f877-11ea-1527-ff4d3db663db
md"## Higher dimensions"

# ╔═╡ aefa84de-f877-11ea-3e26-678008e9739e
md"If we now take columns of the first three rows of the original image, we have vectors in 3D.

A rank-1 matrix corresponds to a line in 3D, while a rank-2 matrix gives a **plane** in 3D. Rank-2 + noise gives a noisy cloud lying close to a plane.

Similarly to what we did above, we need to calculate the ellipsoid that best fits the data. The widths of the axes of the ellipsoid tell us how close to being a line or a plane (rank-1 or rank-2) the data is.
"

# ╔═╡ 0bd9358e-f879-11ea-2c83-ed4e7bf9d903
md"In more than 3D we can no longer visualise the data, but the same idea applies. The calculations are done using the SVD.

If the widths of the ellipsoid in some directions are very small, we can ignore those directions and hence reduce the dimensionality of the data, by changing coordinates to the principal components."

# ╔═╡ eb961e36-f899-11ea-39a9-eb33c949b79d
@bind ϕ1 Slider(0:0.1:180, show_value=true, default=30)

# ╔═╡ fdc87844-f899-11ea-1f2f-afe1cd43a68a
@bind ϕ2 Slider(0:0.1:90, show_value=true, default=30)

# ╔═╡ 232454b4-f87a-11ea-1c69-91edfca1e589
md"## Application: A simple recommendation engine"

# ╔═╡ 2b44df7e-f87a-11ea-1690-dd459eae05a3
md"Suppose we have data on movie recommendations. It might look something like the following data matrix. The rows correspond to different people and the columns to different movies. Missing values are denoted by the special value `missing` and are shown in black.
"

# ╔═╡ e5f67376-f917-11ea-1799-4341e3a758d5
missing

# ╔═╡ e7ae7312-f917-11ea-1276-bf8687cc0e57
typeof(missing)

# ╔═╡ e87e74a6-f87a-11ea-02d5-1970d010bde9
md"""If we think of movies as having properties, or **features**, such as being a drama or action movie, or having a certain actor in a lead role, and that each person has certain preferences, it might be reasonable to think that we should be able to approximate the non-missing part of this matrix by a matrix of low rank, say rank 2. If so, we can then **impute** (fill in) the missing data, and hence give each person a recommendation as to how much they might like the movies that they have not yet seen!"""

# ╔═╡ 63bad6ac-f87b-11ea-23ae-31522dfc74d5
md"""We thus want to find the rank-2 matrix that is *closest* to this given data matrix. "Closest" here requires us to define some kind of distance or **loss function** between the rank-2 matrix and the data matrix, and then to **optimise** this function.

This is one of the applications of the SVD, in the case of a matrix with *no* missing entries. However we can also directly apply **optimisation** methods. The simplest method is **gradient descent**.
"""

# ╔═╡ 2d8b13de-f901-11ea-3198-bb513ea1859c
md"The original matrix and the rank-2 approximation are as follows, with missing data shown in black:"

# ╔═╡ da7592da-f902-11ea-2cee-dbaefacdc382
md"Here we compare the non-missing values. The rank-2 approximation is not too good. We could then increase the rank of the approximation:"

# ╔═╡ 1cf3e098-f864-11ea-3f3a-c53017b73490
md"## Appendix"

# ╔═╡ 2917943c-f864-11ea-3ee6-db952ca7cd67
begin
	show_image(M) = get.(Ref(ColorSchemes.rainbow), M ./ maximum(M))
	show_image(x::AbstractVector) = show_image(x')
end

# ╔═╡ 43bff19e-f864-11ea-2315-0f85b532a325
show_image(flag)

# ╔═╡ 79d2c6f4-f895-11ea-30c4-9d1102c99482
show_image(flag2)

# ╔═╡ b183b6ca-f864-11ea-0b34-4dd3f4f5e69d
show_image(image)

# ╔═╡ 5471ddce-f867-11ea-2519-21981f5ea68b
show_image(noisy_image)

# ╔═╡ 1957f71c-f8eb-11ea-0dcf-339bfa7f96fc
show_image(image[1:2, 1:20])

# ╔═╡ 72bb11b0-f88f-11ea-0e55-b1108300f854
loss(M1, M2) = sum( (M1[i] - M2[i])^2 for i in 1:length(M1) if !ismissing(M2[i]) )

# ╔═╡ feeeb24a-f88f-11ea-287f-219e53615f32
function split_up(v, m, n)
	return v[1:m], v[m+1:m+n], v[m+n+1:2m+n], v[2m+n+1:2m+2n]
end

# ╔═╡ 9822b22e-f89a-11ea-3da9-6199f9de033a
nn = 400

# ╔═╡ 690364dc-f89a-11ea-30e0-d52fbc146ef7
M7 = outer([3, 1, 1], rand(nn)) .+ 0.1 .* randn.();

# ╔═╡ 8b6ea690-f899-11ea-2712-51508ae9c53e
M8 = outer([1, 2, 3], rand(nn)) + outer([-1, 3, -1], rand(nn)) .+ 0.1 .* randn.();

# ╔═╡ 9d5591de-f899-11ea-30d4-b1438066cc92
begin
	scatter(M7[1,:], M7[2,:], M7[3,:], camera=(ϕ1, ϕ2), alpha=0.5, label="rank 1")
	scatter!(M8[1,:], M8[2,:], M8[3,:], camera=(ϕ1, ϕ2), alpha=0.5, label="rank 2")
end

# ╔═╡ c66797fe-f899-11ea-094e-6d65bea15a11
randmissing(n, p_m, dims...) = rand([1:n; [missing for _ ∈ 1:n/(1/p_m -1)]], dims...)

# ╔═╡ 4fa77c96-f87a-11ea-2153-4bdf390369a5
begin
	m, n = 20, 5

	M3 = randmissing(5, 0.4, m, n)
end

# ╔═╡ 0bcc8852-f890-11ea-3715-11cbead7f636
function ff(v, m, n)
	v1, w1, v2, w2 = split_up(v, m, n)

	loss(outer(v1, w1) + outer(v2, w2), M3)
end

# ╔═╡ 7040dc72-f893-11ea-3d22-4fbd452faa41
ff2(v) = ff(v, m, n)

# ╔═╡ 20e94d56-f890-11ea-3953-cbd70cec8ebd
total = 2(m + n)

# ╔═╡ 1dbcf15a-f890-11ea-008c-8935edfbdb1c
ff(rand(total), m, n)

# ╔═╡ 1d7e264c-f891-11ea-131d-134cbfff1ac0
function optimise(f)
	x = rand(total)
	η = 0.01

	for i in 1:1000
		x -= η * ForwardDiff.gradient(f, x)
	end

	return x
end

# ╔═╡ 7715e100-f893-11ea-3768-f9a59d8cc06c
begin
	xxx = optimise( v -> ff(v, m, n) )

	(v1, w1, v2, w2) = split_up(xxx, m, n)
	M4 = outer(v1, w1) + outer(v2, w2)
end

# ╔═╡ 53819d1e-f902-11ea-3388-ebc082de7053
begin
	M4_new = float.(M3)
	indices = (!ismissing).(M3)
	M4_new[indices] .= M4[indices]
end

# ╔═╡ 49b83854-f894-11ea-1f07-c95929bf9aea
M5 = replace(M3, missing=>0)

# ╔═╡ 81115c14-f893-11ea-0147-c9fd45b7b777
show_image(replace(M3, missing=>0))

# ╔═╡ 9da0cbaa-f893-11ea-2d94-951c1f947a2d
let
	global M6 = copy(M4)
	M6[ismissing.(M3)] .= 0

	M6
end

# ╔═╡ 1857c66c-f894-11ea-0ba2-efe85cd442aa
[show_image(M5)', show_image(M6)']

# ╔═╡ 54ff9624-f901-11ea-309f-e396acc96f23
function show_image_missing(M)
	colors = show_image(replace(M, missing=>0))
	colors[ismissing.(M)] .= RGB(0, 0, 0)

	colors
end

# ╔═╡ 50d80f8c-f900-11ea-27fb-c5a453928534
show_image_missing(M3)

# ╔═╡ 31ed8eac-f901-11ea-3443-25c0c459803c
[show_image_missing(M3)', show_image_missing(M4)']

# ╔═╡ a0e357a0-f902-11ea-1895-651d395d025d
[show_image_missing(M3)', show_image_missing(M4_new)']

# ╔═╡ b0fadede-f901-11ea-2248-ab8049d719f5
M'

# ╔═╡ 8f599fae-f901-11ea-25e5-11a1f569aef1
show_image_missing(M3)

# ╔═╡ e465ca72-f901-11ea-22f3-318147c8d79a
colors = show_image(replace(M3, missing=>0))

# ╔═╡ f175b60a-f901-11ea-0fcb-01fc17ec2a97
colors[ismissing.(M3)]

# ╔═╡ Cell order:
# ╟─3b9941ac-6043-4dc6-850f-4c7b3ae9d9a7
# ╟─7365084a-1f37-4897-bca4-fc5855c5ee4e
# ╟─ed7ff6b2-f863-11ea-1a59-eb242a8674e3
# ╟─fed5845e-f863-11ea-2f95-c331d3c62647
# ╠═0e1a6d80-f864-11ea-074a-5f7890180114
# ╠═2e497e30-f895-11ea-09f1-d7f2c1f61193
# ╟─ab3d55cc-f905-11ea-2f22-5398f3aca803
# ╠═13b6c108-f864-11ea-2447-2b0741f15c7b
# ╠═e66b30a6-f914-11ea-2c0f-35282d45a30a
# ╠═43bff19e-f864-11ea-2315-0f85b532a325
# ╠═71d1b12e-f895-11ea-39df-f5c18a7766c3
# ╠═79d2c6f4-f895-11ea-30c4-9d1102c99482
# ╟─cdbe1d8e-f905-11ea-3884-efeeef386dda
# ╟─d9aa9af0-f865-11ea-379e-f16b452bd94c
# ╟─2e8ae92a-f867-11ea-0219-1bdd9627c1ea
# ╠═38adc490-f867-11ea-1de5-3b633aff7c97
# ╠═b183b6ca-f864-11ea-0b34-4dd3f4f5e69d
# ╟─9cf23f9a-f864-11ea-3a08-af448aceefd8
# ╟─a5b62530-f864-11ea-21e8-71ccfed487f8
# ╠═5471ddce-f867-11ea-2519-21981f5ea68b
# ╟─c41df86c-f865-11ea-1253-4942bbdbe9d2
# ╟─7fca33ac-f864-11ea-2a8b-933eb382c172
# ╟─283f5da4-f866-11ea-27d4-957ca2551b92
# ╠═1957f71c-f8eb-11ea-0dcf-339bfa7f96fc
# ╠═54977286-f908-11ea-166d-d1df33f38454
# ╠═7b4e90b4-f866-11ea-26b3-95efde6c650b
# ╟─f574ad7c-f866-11ea-0efa-d9d0602aa63b
# ╟─8775b3fe-f866-11ea-3e6f-9732e39a3525
# ╠═0dcfd858-f867-11ea-301c-c3ca0a224117
# ╠═7bacf44e-f896-11ea-38be-2b16ae7ca99f
# ╟─1147cbda-f867-11ea-08fa-ef6ed2ae1e93
# ╟─8a611e36-f867-11ea-121f-317b7c145fe3
# ╟─f7371934-f867-11ea-3b53-d1566684585c
# ╟─119dc35c-ec94-11ea-190c-23a750fbe7f4
# ╟─1e058ba2-ec94-11ea-09af-7f9f9cc3a233
# ╠═2043d4e6-ec94-11ea-1e1a-c75742eafe71
# ╟─2a705962-ec94-11ea-1181-2f001ccf472f
# ╟─987c1f2e-f868-11ea-1125-0d8c02843ae4
# ╟─9e78b048-f868-11ea-192e-d903265d1eb5
# ╟─24df1f32-ec90-11ea-1f6d-03c1bfa5df8e
# ╠═13f6ccac-7ce0-48d7-a0ef-e83489625e1d
# ╠═aec46a9b-f743-4cbd-97a7-3ef3cac78b12
# ╟─1b8c743e-ec90-11ea-10aa-e3b94f768f82
# ╟─f5358ce4-f86a-11ea-2989-b1f37be89183
# ╟─870d3efa-f8fc-11ea-1593-1552511dcf86
# ╟─03ab44c0-f8fd-11ea-2243-1f3580f98a65
# ╠═2c3721da-f86b-11ea-36cf-3fe4c6622dc6
# ╟─6dec0db8-ec93-11ea-24ad-e17870ee64c2
# ╟─5fab2c32-f86b-11ea-2f27-ed5feaac1fa5
# ╟─ae9a2900-ec93-11ea-1ae5-0748221328fc
# ╟─b81c9db2-ec93-11ea-0dbd-4bd0951cb2cc
# ╟─80722856-f86d-11ea-363d-53fc5f6b8152
# ╟─b8fa6a1c-f86d-11ea-3d6b-2959d737254b
# ╟─3547f296-f86f-11ea-1698-53d3c1a0bc30
# ╟─7a83101e-f871-11ea-1d87-4946162777b5
# ╟─e8276b4e-f86f-11ea-38be-218a72452b10
# ╟─3b71142c-f86f-11ea-0d43-47011d00786c
# ╠═d71fdaea-f86f-11ea-1a1f-45e4d50926d3
# ╠═78763674-f8fe-11ea-349c-d997f30ac1f6
# ╠═757c6808-f8fe-11ea-39bb-47e4da65113a
# ╟─88bbe1bc-f86f-11ea-3b6b-29175ddbea04
# ╟─cd9e05ee-f86f-11ea-0422-25f8329c7ef2
# ╟─7eb51908-f906-11ea-19d2-e947d81cb743
# ╠═4f1980ea-f86f-11ea-3df2-35cca6c961f3
# ╟─2ffe7ed0-f870-11ea-06aa-390581500ca1
# ╟─a5cdad52-f906-11ea-0486-755a6403a367
# ╟─0115c974-f871-11ea-1204-054510848849
# ╠═0935c870-f871-11ea-2a0b-b1b824379350
# ╟─e4af4d26-f877-11ea-1de3-a9f8d389138e
# ╟─bf57f674-f906-11ea-08eb-9b50818a025b
# ╠═17e015fe-f8ff-11ea-17b4-a3aa072cd7b3
# ╟─045b9b98-f8ff-11ea-0d49-5b209319e951
# ╟─cfec1ec4-f8ff-11ea-265d-ab4844f0f739
# ╟─e6e900b8-f904-11ea-2a0d-953b99785553
# ╟─aaff88e8-f877-11ea-1527-ff4d3db663db
# ╟─aefa84de-f877-11ea-3e26-678008e9739e
# ╟─0bd9358e-f879-11ea-2c83-ed4e7bf9d903
# ╠═690364dc-f89a-11ea-30e0-d52fbc146ef7
# ╠═8b6ea690-f899-11ea-2712-51508ae9c53e
# ╠═eb961e36-f899-11ea-39a9-eb33c949b79d
# ╠═fdc87844-f899-11ea-1f2f-afe1cd43a68a
# ╟─9d5591de-f899-11ea-30d4-b1438066cc92
# ╟─232454b4-f87a-11ea-1c69-91edfca1e589
# ╟─2b44df7e-f87a-11ea-1690-dd459eae05a3
# ╠═e5f67376-f917-11ea-1799-4341e3a758d5
# ╠═e7ae7312-f917-11ea-1276-bf8687cc0e57
# ╠═4fa77c96-f87a-11ea-2153-4bdf390369a5
# ╠═50d80f8c-f900-11ea-27fb-c5a453928534
# ╟─e87e74a6-f87a-11ea-02d5-1970d010bde9
# ╟─63bad6ac-f87b-11ea-23ae-31522dfc74d5
# ╟─2d8b13de-f901-11ea-3198-bb513ea1859c
# ╠═31ed8eac-f901-11ea-3443-25c0c459803c
# ╟─53819d1e-f902-11ea-3388-ebc082de7053
# ╟─da7592da-f902-11ea-2cee-dbaefacdc382
# ╠═a0e357a0-f902-11ea-1895-651d395d025d
# ╟─1cf3e098-f864-11ea-3f3a-c53017b73490
# ╠═35e83a04-f864-11ea-0a8e-9ddf6eec02f3
# ╠═2917943c-f864-11ea-3ee6-db952ca7cd67
# ╠═72bb11b0-f88f-11ea-0e55-b1108300f854
# ╠═feeeb24a-f88f-11ea-287f-219e53615f32
# ╠═0bcc8852-f890-11ea-3715-11cbead7f636
# ╠═7040dc72-f893-11ea-3d22-4fbd452faa41
# ╠═1dbcf15a-f890-11ea-008c-8935edfbdb1c
# ╠═20e94d56-f890-11ea-3953-cbd70cec8ebd
# ╠═16887070-f891-11ea-2db3-47b91930e728
# ╠═1d7e264c-f891-11ea-131d-134cbfff1ac0
# ╠═7715e100-f893-11ea-3768-f9a59d8cc06c
# ╠═49b83854-f894-11ea-1f07-c95929bf9aea
# ╠═81115c14-f893-11ea-0147-c9fd45b7b777
# ╠═9da0cbaa-f893-11ea-2d94-951c1f947a2d
# ╠═1857c66c-f894-11ea-0ba2-efe85cd442aa
# ╠═9822b22e-f89a-11ea-3da9-6199f9de033a
# ╠═c66797fe-f899-11ea-094e-6d65bea15a11
# ╠═54ff9624-f901-11ea-309f-e396acc96f23
# ╠═b0fadede-f901-11ea-2248-ab8049d719f5
# ╠═8f599fae-f901-11ea-25e5-11a1f569aef1
# ╠═e465ca72-f901-11ea-22f3-318147c8d79a
# ╠═f175b60a-f901-11ea-0fcb-01fc17ec2a97