using LinearAlgebra
using Random
using Statistics
using DataFrames
using FreqTables

#:::::::::::::::::::::::::::::::::::::::::::::::::::
# question 1
#:::::::::::::::::::::::::::::::::::::::::::::::::::
using Optim
f(x) = -x[1]^4-10x[1]^3-2x[1]^2-3x[1]-2
minusf(x) = x[1]^4+10x[1]^3+2x[1]^2+3x[1]+2
startval = rand(1)   # random starting value
result = optimize(minusf, startval, BFGS())

#:::::::::::::::::::::::::::::::::::::::::::::::::::
# question 2
#:::::::::::::::::::::::::::::::::::::::::::::::::::
using DataFrames
using CSV
using HTTP
url = "https://raw.githubusercontent.com/OU-PhD-Econometrics/fall-2020/master/ProblemSets/PS1-julia-intro/nlsw88.csv"
df = CSV.read(HTTP.get(url).body,DataFrame)
X = [ones(size(df,1),1) df.age df.race.==1 df.collgrad.==1]
y = df.married.==1

function ols(beta, X, y)
    ssr = (y.-X*beta)'*(y.-X*beta)
    return ssr
end

beta_hat_ols = optimize(b -> ols(b, X, y), rand(size(X,2)), LBFGS(), Optim.Options(g_tol=1e-6, iterations=100_000, show_trace=true))
println(beta_hat_ols.minimizer)

using GLM
bols = inv(X'*X)*X'*y
df.white = df.race.==1
bols_lm = lm(@formula(married ~ age + white + collgrad), df)

#:::::::::::::::::::::::::::::::::::::::::::::::::::
# question 3
#:::::::::::::::::::::::::::::::::::::::::::::::::::
function logit(alpha, X, d)
    loglike = sum([d[i]*log(exp(sum(alpha.*X[i,:]))/(1+exp(sum(alpha.*X[i,:]))))+(1-d[i])*log(1/(1+exp(sum(alpha.*X[i,:])))) for i=1:size(X,1)])
    return loglike
end

startval = rand(size(X,2))
obj = optimize(alpha->-logit(alpha, X, y), startval, LBFGS())
println(obj.minimizer)

#:::::::::::::::::::::::::::::::::::::::::::::::::::
# question 4
#:::::::::::::::::::::::::::::::::::::::::::::::::::
# see Lecture 3 slides for example
res = glm(X, y, Binomial(), LogitLink())
println(res)

#:::::::::::::::::::::::::::::::::::::::::::::::::::
# question 5
#:::::::::::::::::::::::::::::::::::::::::::::::::::
freqtable(df, :occupation) # note small number of obs in some occupations
df = dropmissing(df, :occupation)
df[df.occupation.==8 ,:occupation] .= 7
df[df.occupation.==9 ,:occupation] .= 7
df[df.occupation.==10,:occupation] .= 7
df[df.occupation.==11,:occupation] .= 7
df[df.occupation.==12,:occupation] .= 7
df[df.occupation.==13,:occupation] .= 7
freqtable(df, :occupation) # problem solved

X = [ones(size(df,1),1) df.age df.race.==1 df.collgrad.==1]
y = df.occupation

function mlogit(alpha, X, d)
    loglike = 0
    for i = 1:size(X,1)
        term1 = sum([convert(Float64,d[i]==j)*sum(alpha[:,j].*X[i,:]) for j =1:6])
        term2 = log(1 + sum([exp(sum(alpha[:,j].*X[i,:])) for j =1:6]))
        loglike += term1 - term2
    end
    return loglike
end

#startval = zeros(size(X,2),6)
#startval = rand(size(X,2),6)
startval = 2*rand(size(X,2),6).-1
obj = optimize(alpha->-mlogit(alpha, X, y), startval, LBFGS(), Optim.Options(g_tol=1e-5, iterations=100000))
println(obj.minimizer)


"""
    dirichlet_beta()

 Calculates Dirichlet beta function,
     [https://en.wikipedia.org/wiki/Dirichlet_beta_function](https://en.wikipedia.org/wiki/Dirichlet_beta_function)

## Arguments
* ``s`` `::Number`: it should work for any type of number, but mainly tested for `Complex{Float64}`

## Examples
```jldoctest; setup = :(using Polylogarithms)
julia> dirichlet_beta(1.5)
0.8645026534612017
```
"""
function dirichlet_beta(s::Number)
    β = 4.0^(-s) * ( SpecialFunctions.zeta(s,0.25) - SpecialFunctions.zeta(s,0.75) )
end


using AdFem
using PyPlot
using SparseArrays

function f1func(x,y)
    18.8495559215388*pi^2*sin(pi*x)^2*sin(pi*y)*cos(pi*y) - 6.28318530717959*pi^2*sin(pi*y)*cos(pi*x)^2*cos(pi*y) + pi*sin(pi*y)*cos(pi*x)
end
function f2func(x,y)
    -18.8495559215388*pi^2*sin(pi*x)*sin(pi*y)^2*cos(pi*x) + 6.28318530717959*pi^2*sin(pi*x)*cos(pi*x)*cos(pi*y)^2 + pi*sin(pi*x)*cos(pi*y)
end

m = 50
n = 50
h = 1/n
mmesh = Mesh(m, n, h, degree=2)
ν = 0.5
K = ν*constant(compute_fem_laplace_matrix(mmesh))
B = constant(compute_interaction_matrix(mmesh))
Z = [K -B'
    -B spzero(size(B,1))]

bd = bcnode(mmesh)
bd = [bd; bd .+ mmesh.ndof; 2mmesh.ndof + 1]

F1 = eval_f_on_gauss_pts(f1func, mmesh)
F2 = eval_f_on_gauss_pts(f2func, mmesh)
F = compute_fem_source_term(F1, F2, mmesh)
rhs = [F;zeros(mmesh.nelem)]
Z, rhs = impose_Dirichlet_boundary_conditions(Z, rhs, bd, zeros(length(bd)))
sol = Z\rhs

sess = Session(); init(sess)
S = run(sess, sol)

xy = fem_nodes(mmesh)
x, y = xy[:,1], xy[:,2]
U = @. 2*pi*sin(pi*x)*sin(pi*x)*cos(pi*y)*sin(pi*y)
figure(figsize=(12,5))
subplot(121)
visualize_scalar_on_fem_points(U, mmesh)
title("Reference")
subplot(122)
visualize_scalar_on_fem_points(S[1:mmesh.nnode], mmesh)
title("Computed")
savefig("stokes1.png")

U = @. -2*pi*sin(pi*x)*sin(pi*y)*cos(pi*x)*sin(pi*y)
figure(figsize=(12,5))
subplot(121)
visualize_scalar_on_fem_points(U, mmesh)
title("Reference")
subplot(122)
visualize_scalar_on_fem_points(S[mmesh.ndof+1:mmesh.ndof+mmesh.nnode], mmesh)
title("Computed")
savefig("stokes2.png")

xy = fvm_nodes(mmesh)
x, y = xy[:,1], xy[:,2]
p = @. sin(pi*x)*sin(pi*y)
figure(figsize=(12,5))
subplot(121)
visualize_scalar_on_fvm_points(p, mmesh)
title("Reference")
subplot(122)
visualize_scalar_on_fvm_points(S[2mmesh.ndof+1:end], mmesh)
title("Computed")
savefig("stokes3.png")


function acqr_kfilter(y,a,c,q,r,x10,P10)
	#
	# Kalman filter for model
	#
	# x_{t+1} = a*x_{t} + w_{t}
	# y_{t}   = c*x_{t} + v_{t}
	#
	# cov(w_{t},v_{t}) = [q 0;0 r]
	#
	# javier.cara@upm.es, 2020-05
	#

	nt = length(y)

	# allocation
	xtt = zeros(nt)
	Ptt = zeros(nt)
	xtt1 = zeros(nt+1)
	Ptt1 = zeros(nt+1)
	et = zeros(nt)
	St = zeros(nt)
	Kt = zeros(nt)
	loglik = 0.0

	# Filter
	xtt1[1] = x10
	Ptt1[1] = P10
	for t in 1:nt
		#  innovations
		et[t] = y[t] - c*xtt1[t]
		St[t] = c^2*Ptt1[t] + r # et variance

		# Kalman gain
		Kt[t] = Ptt1[t]*c/St[t] # kalman gain

		# filtered values
		xtt[t] = xtt1[t] + Kt[t]*et[t]
		Ptt[t] = (1 - Kt[t]*c)*Ptt1[t]

		# one-step ahead prediction
		xtt1[t+1] = a*xtt[t]
		Ptt1[t+1] = a^2*Ptt[t] + q

		# likelihood
		#l0 = et[t]^2/St[t] # typeof(l0) = Array{Float64,1}
		loglik = loglik + log(St[t]) + et[t]^2/St[t]
	end

	loglik =  - nt/2*log(2*pi) - 0.5*loglik

	return xtt,Ptt,xtt1,Ptt1,et,St,Kt,loglik

end


# # Maximum Cut Problem
#
# We are interested in solving an approximation to the maximum cut problem using semidefinite programming.
# Consider the graph $G(V,E)$ with weights $w_{ij}$ shown below:

## create the weighted adjacency matrix of the graph and plot the graph
using LinearAlgebra, Plots, GraphRecipes
n = 4;
W = zeros(n, n);
W[1, 2] = 1; W[1, 4] = 8;
W[2, 3] = 2; W[2, 4] = 10;
W[3, 4] = 6;
W = Symmetric(W)
graphplot(W, names = 1:n, edgelabel = W,  x = [0; 1; 1; 0], y = [1; 1; 0; 0], fontsize = 12, nodeshape =:circle)

# The maximum cut problem tries to find a cut or a partition of the graph's vertices into two complementary sets $S$ and $\bar{S}$ such that the total weight of the edges between the two sets is maximized. For this small graph the problem is trivial. The optimal solution is $S = \{1,2,3 \}$ and $\bar{S}=\{4\}$.
# Formally, this problem can be written as a mixed-integer optimisation problem:
# $$
# \begin{array}{ll} \text{maximize} &  \frac{1}{2} \sum_{i < j} w_{ij}(1 - y_i y_j)\\
# \text{subject to} &  y_i \in \, \{-1, 1 \}, \quad \forall i \in V,
# \end{array}
# $$
# where $y_i = 1$ indicates that $v_i \in S$ and $y_i = -1$ indicates that $v_i \in \bar{S}$. This problem is of interest in the field of integrated circuit design, where one tries to minimize the number of cross-layer connections in a circuit under layout constraints.

# For more complicated graphs this problem quickly becomes hard to solve to optimality and in fact is known to be NP-hard. For this example we are interested in the randomized approximation algorithm that relaxes the problem to an SDP and was devised in Goemans and Williamson (1995) \[1\] (see for more details).

# The approach can be divided into three steps:
# 1. Solve a relaxed SDP to obtain $Y^*$
# 2. Recover an approximate solution $V$ via a Cholesky factorisation $Y^* = V^\top V$
# 3. Round the approximate solution using a random vector $r$ from the unit sphere.
# The authors showed that this approach guarantees a solution of at least $0.87856$ times the optimal solution.

# ## Solving the primal SDP
# Before we formulate the SDP, let's compute the Laplacian matrix $L$:
using LinearAlgebra, Random, SparseArrays, COSMO, JuMP

rng = Random.MersenneTwister(1)

## Compute the Laplacian matrix of the graph
L = diagm(0 => W * ones(n)) - W;

# Given the Laplacian matrix $L$, we are looking for the optimal $Y^*$ that solves the following SDP:
# $$
# \begin{array}{ll} \text{maximize} &  \frac{1}{4} \langle L, Y \rangle \\
# \text{subject to} & Y_{ii} = 1 \quad \text{for } i = 1,\dots,n\\
#                    & Y \in \mathbf{S}_{+}^n.
# \end{array}
# $$
# Notice that the solution $Y^*$ can be viewed as a correlation matrix. Let's solve the problem using COSMO and JuMP.

#-
model = JuMP.Model(COSMO.Optimizer);
@variable(model, Y[1:n, 1:n], PSD);
@objective(model, Max, 1 / 4 * dot(L, Y));
@constraint(model, [i = 1:n], Y[i, i] == 1.);
JuMP.optimize!(model)

#-
Yopt = JuMP.value.(Y);
obj_val = JuMP.objective_value(model)

# ## Solving the dual SDP
# Notice that the decision matrix $Y$ is generally dense (as correctly classified in the solver output above). Therefore, we won't be able to utilize COSMO's chordal decomposition features. However, assuming strong duality, it turns out that we can also solve the dual problem, which is given by:
# $$
# \begin{array}{ll} \text{minimize} &  \sum_i \gamma_i \\
# \text{subject to} & \text{diag}(\gamma) - \frac{1}{4} L = S \\
#                   & S \in \mathbf{S}_{+}^n.
# \end{array}
# $$
# As you can see, the matrix $S$ is constrained to have zeros in places specified by the graph (Laplacian). Therefore, COSMO can try to decompose this SDP (see solver output) and speed up its algorithm:

#-
model_dual = JuMP.Model(optimizer_with_attributes(COSMO.Optimizer, "complete_dual" => true));
@variable(model_dual, γ[1:n]);
@objective(model_dual, Min,  sum(γ));
@constraint(model_dual, lmi, Symmetric(-1/4 * L + diagm(γ)) in JuMP.PSDCone());
JuMP.optimize!(model_dual)
obj_val = JuMP.objective_value(model_dual)

# The primal variable $Y^*$ can be recovered from the dual variable associated with the LMI-constraint:
#-
Yopt = dual(lmi)
# To get the correct positive semidefinite dual variable, we have to enable PSD completion of the dual variable in COSMO. Now, that we have a solution we can perform the remaining steps in the approximation algorithm.

# ## Cholesky factorisation of Y
# Compute the Cholesky factorisation of $Y = V^\top V$ to find the unit vectors $v_1, \dots, v_n$.

#-
factor = cholesky(Yopt, Val(true); check = false);
V = Matrix((factor.P * factor.L)');
## normalize columns
for i in 1:n
    V[:, i] ./= norm(V[:, i]);
end

# ## Rounding the approximate solution
# It remains to round the unit vectors $v_i$ using a random vector $r$ with each component drawn from $\mathcal{U}(0, 1)$, to obtain the $y_i$'s:

#-
r = rand(rng, n)
r /= norm(r, 2)
y = ones(n)
for i in 1:n
  dot(r, V[:, i]) <= 0 && (y[i] = -1)
end
y
# For larger graphs this rounding step could be repeated several times to improve the rounding.
# As expected $S = \{1, 2, 3\}$ and $\bar{S}=\{ 4\}$.
# ## References
# [1] Goemans and Williamson - Improved Approximation Algorithms for Maximum Cut and Satisfiability Problems Using Semidefinite Programs (1995)

# [2] Post-processing code steps 2 and 3 from this [JuMP example](https://github.com/jump-dev/JuMP.jl/blob/master/examples/max_cut_sdp.jl)


f(u,p,t) = 0.98u

using DifferentialEquations
f(u,p,t) = 0.98u
u0 = 1.0
tspan = (0.0,1.0)
prob = ODEProblem(f,u0,tspan)

sol = solve(prob)

using Plots; gr()
plot(sol)

plot(sol,linewidth=5,title="Solution to the linear ODE with a thick line",
     xaxis="Time (t)",yaxis="u(t) (in μm)",label="My Thick Line!") # legend=false

plot!(sol.t, t->1.0*exp(0.98t),lw=3,ls=:dash,label="True Solution!")

sol.t

sol.u

[t+u for (u,t) in tuples(sol)]

sol

sol(0.45)

sol = solve(prob,abstol=1e-8,reltol=1e-8)

plot(sol)
plot!(sol.t, t->1.0*exp(0.98t),lw=3,ls=:dash,label="True Solution!")

sol = solve(prob,saveat=0.1)

sol = solve(prob,saveat=[0.2,0.7,0.9])

sol = solve(prob,dense=false)

sol = solve(prob,save_everystep=false)

sol = solve(prob,save_everystep=false,save_start = false)

sol = solve(prob,alg_hints=[:stiff])

sol = solve(prob,Tsit5(),reltol=1e-6)

function lorenz!(du,u,p,t)
    σ,ρ,β = p
    du[1] = σ*(u[2]-u[1])
    du[2] = u[1]*(ρ-u[3]) - u[2]
    du[3] = u[1]*u[2] - β*u[3]
end

u0 = [1.0,0.0,0.0]

p = (10,28,8/3) # we could also make this an array, or any other type!

tspan = (0.0,100.0)
prob = ODEProblem(lorenz!,u0,tspan,p)

sol = solve(prob)

sol.t[10],sol[10]

sol[2,10]

A = Array(sol)

plot(sol)

plot(sol,vars=(1,2,3))

plot(sol,vars=(1,2,3),denseplot=false)

plot(sol,vars=(0,2))

function lotka_volterra!(du,u,p,t)
  du[1] = p[1]*u[1] - p[2]*u[1]*u[2]
  du[2] = -p[3]*u[2] + p[4]*u[1]*u[2]
end

using ParameterizedFunctions
lv! = @ode_def LotkaVolterra begin
  dx = a*x - b*x*y
  dy = -c*y + d*x*y
end a b c d

u0 = [1.0,1.0]
p = (1.5,1.0,3.0,1.0)
tspan = (0.0,10.0)
prob = ODEProblem(lv!,u0,tspan,p)
sol = solve(prob)
plot(sol)

lv!.Jex

A  = [1. 0  0 -5
      4 -2  4 -3
     -4  0  0  1
      5 -2  2  3]
u0 = rand(4,2)
tspan = (0.0,1.0)
f(u,p,t) = A*u
prob = ODEProblem(f,u0,tspan)
sol = solve(prob)

sol[3]

big_u0 = big.(u0)

prob = ODEProblem(f,big_u0,tspan)
sol = solve(prob)

sol[1,3]

prob = ODEProblem(f,big_u0,big.(tspan))
sol = solve(prob)

using StaticArrays
A  = @SMatrix [ 1.0  0.0 0.0 -5.0
                4.0 -2.0 4.0 -3.0
               -4.0  0.0 0.0  1.0
                5.0 -2.0 2.0  3.0]
u0 = @SMatrix rand(4,2)
tspan = (0.0,1.0)
f(u,p,t) = A*u
prob = ODEProblem(f,u0,tspan)
sol = solve(prob)

sol[3]

using DiffEqTutorials
DiffEqTutorials.tutorial_footer(WEAVE_ARGS[:folder],WEAVE_ARGS[:file])


function romberg(f::Function, a::Number, b::Number, nl::Int)
  #= Numerical Integration of functions using Romberg's method.
  Keyword arguments:
  f -- Anonymous functon to integrate
  a -- Lower limit of integration
  b -- Upper limit of integration
  nl -- The level of integration to go to.

  Returns:
  The result of the integration of f between a & b
  =#

  intTable = zeros(nl, nl); # Empty matrix to use as div diff table

  # Calculate the first column of the div diff table using the trapezium rule.
  h = (b - a);
  for k = 1:nl
    intTable[k,1] = 0.5 * h * (f(a) + 2*sum(f([a+h:h:b-h])) + f(b));
    h /= 2;
  end

  # Calculate the remaining elements using Romberg's method.
  for k = 2:nl, j = nl-k+1:-1:1
    intTable[j, k] = ((4^(k-1))intTable[j+1, k-1] - intTable[j, k-1])/((4^(k-1)) - 1);
  end

  return intTable[1, end];
end

function romberg(intEstimates::Array{Float64,1})
  #= Estiamte integrals from sets of data using the Romberg Method.

  Keyword arguments:
  intEstimates -- An array of initial estimates of integrals

  Returns:
    A tuple containing the integral and the relative error in the estimate
  =#
  intTable = zeros(length(intEstimates), length(intEstimates));
  relativeErrorTable = zeros(size(intEstimates));

  intTable[:, 1] = intEstimates;

  for k = 2:size(intTable)[1]
    for j = size(intTable)[1]-k+1:-1:1
      intTable[j, k] = ((4^(k-1))intTable[j+1, k-1] - intTable[j, k-1])/((4^(k-1)) - 1);
    end
  end

  for k = 2:size(intTable)[1]
    relativeErrorTable[k, 1] = 100 * abs((intTable[1, k] - intTable[2, k-1]) / intTable[1, k]);
  end

  return (intTable[1, end], relativeErrorTable[1, end]);
end


using CompScienceMeshes
using SauterSchwabQuadrature

pI = point(1,5,3)
pII = point(2,5,3)
pIII = point(7,1,0)
pIV = point(5,1,-3)

Sourcechart = simplex(pII,pI,pIII)
Testchart = simplex(pII,pI,pIV)

Accuracy = 12

function integrand(x,y)
			return(((x-pI)'*(y-pII))*exp(-im*1*norm(x-y))/(4pi*norm(x-y)))
end

result = sauterschwabintegral(Sourcechart, Testchart, integrand, Accuracy, Accuracy)
println(result)

#=For those who want to test the sauterschwab_nonparameterized() function,
may uncomment the following five lines=#

#sourcechart = simplex(pI,pIII,pII)
#testchart = simplex(pI,pIV,pII)
#ce = CommonEdge(Accuracy)
#result2 = sauterschwab_nonparameterized(sourcechart, testchart, integrand, ce)
#println(result2)

#=The first argument and the third argument of both simplex() functions are
equal, hence the required conditions are fulfilled. The user may also compare
the two results, and see that both are equal=#


export Strang
"""
`Strang` matrix

A special `SymTridiagonal` matrix named after Gilbert Strang

```julia
julia> Strang(6)
6x6 Strang{Float64}:
  2.0  -1.0   0.0   0.0   0.0   0.0
 -1.0   2.0  -1.0   0.0   0.0   0.0
  0.0  -1.0   2.0  -1.0   0.0   0.0
  0.0   0.0  -1.0   2.0  -1.0   0.0
  0.0   0.0   0.0  -1.0   2.0  -1.0
  0.0   0.0   0.0   0.0  -1.0   2.0
```
"""
struct Strang{T} <: AbstractArray{T, 2}
    n :: Int
end
Strang(n::Int) = Strang{Float64}(n)
strang(T, n)= n >1 ? SymTridiagonal(2ones(T, n),-ones(T, n-1)) :
              n==1 ? Diagonal([2one(T)]) : error("Invalid dimension ", n)
function getindex(S::Strang{T}, i, j) where T
    i == j && return 2
    abs(i - j) == 1 && return -1
    0
end
getindex(S::Strang{T}, I...) where T = getindex(S,ind2sub(size(S),I...)...)
size(S::Strang, r::Int) = r==1 || r==2 ? S.n : throw(ArgumentError("Invalid dimension $r"))
size(S::Strang) = S.n, S.n
Matrix(S::Strang{T}) where T = Matrix(strang(T, S.n))
*(A::VecOrMat{T}, S::Strang{T}) where T = A*Matrix(S)
*(S::Strang{T}, A::VecOrMat{T}) where T = Matrix(S)*A


# Summary statistics

"""
    genvar(X)

Compute the generalized sample variance of `X`. If `X` is a vector, one-column matrix,
or other one-dimensional iterable, this is equivalent to the sample variance.
Otherwise if `X` is a matrix, this is equivalent to the determinant of the covariance
matrix of `X`.

!!! note
    The generalized sample variance will be 0 if the columns of the matrix of deviations
    are linearly dependent.
"""
genvar(X::AbstractMatrix) = size(X, 2) == 1 ? var(vec(X)) : det(cov(X))
genvar(itr) = var(itr)

"""
    totalvar(X)

Compute the total sample variance of `X`. If `X` is a vector, one-column matrix,
or other one-dimensional iterable, this is equivalent to the sample variance.
Otherwise if `X` is a matrix, this is equivalent to the sum of the diagonal elements
of the covariance matrix of `X`.
"""
totalvar(X::AbstractMatrix) = sum(var(X, 1))
totalvar(itr) = var(itr)


module EllipsoidAlgebra

using Plots, LinearAlgebra

"""
E(x, P) = { z : (z-x) P ( z-x) <= 1
"""
struct Ellipsoid{Tx, TP}
    x0::Tx
    P::TP
end

Ellipsoid(x0, P, V) = Ellipsoid(x0, P/V)

function plotEllipse!(e::Ellipsoid; kwargs...)

    @assert length(e.x0) == 2

    Pinv_half = real.(sqrt(inv(e.P)))

    x(θ) = e.x0[1] + first(Pinv_half * [cos(θ), sin(θ)])
    y(θ) = e.x0[2] + last(Pinv_half * [cos(θ), sin(θ)])

    plot!(x, y, 0, 2π; kwargs...)

end

function project(e::Ellipsoid, dims=[1,2])

    N = length(e.x0)
    Pinv_half = sqrt(inv(e.P))

    # construct projection matrix
    T = zeros(2, N)
    T[1,dims[1]] = 1
    T[2, dims[2]] = 1

    # get new shape matrix
    TPinvhalf = T * Pinv_half
    PPinv = TPinvhalf * TPinvhalf'
    Pp = real.(inv(PPinv))

    # get new center
    xp0 = T * e.x0

    return Ellipsoid(xp0, Pp)

end

function sampleRand(e::Ellipsoid, N=1)

    @assert isposdef(e.P)

    Pinv_half = sqrt(inv(e.P))

    # sample random from unit sphere, and map to ellipsoid
     y = 2*rand(length(e.x0), N) .- 1
     for i=1:N
        while norm(y[:,i]) > 1.0
            y[:,i] = 2*rand(length(e.x0), 1) .- 1
        end

    end

    xs = similar(y)
    for i=1:N
        xs[:,i] = e.x0 + Pinv_half * y[:,i]
    end

    return xs
end

function creategrid(d::Integer, n::Integer)

    @assert d >= 1 ("d (number of dimensions) must be a positive integer")
    @assert n >= 2 ("n (number of points) must be a at least 2")

    r = range(-1, 1, length = n)

    iter = Iterators.product((r for _ in 1:d)...)

    return vec([collect(i) for i in iter if dot(i, i) <= 1])
end

const sphere_6_grid = creategrid(6, 5)

function sampleUniformGrid(e::Ellipsoid, n=5)
    N = length(e.x0)
    Pinv_half = sqrt(inv(e.P))
    if N == 6
        pts = [e.x0 + Pinv_half * i for i in sphere_6_grid]
        return pts
    else
        pts = [e.x0 + Pinv_half * i for i in creategrid(N, n)]
        return pts
    end
end

end


@doc raw"""
    MaternKernel([ν=1 [,θ=1]])

The Matern kernel is a Mercer kernel given by:

```math
\kappa(\mathbf{x},\mathbf{y}) =
\frac{1}{2^{\nu-1}\Gamma(\nu)}
\left(\frac{2\sqrt{\nu}||\mathbf{x}-\mathbf{y}||}{\theta}\right)^{\nu}
K_{\nu}\left(\frac{2\sqrt{\nu}||\mathbf{x}-\mathbf{y}||}{\theta}\right)
```

where ``\Gamma`` is the gamma function, ``K_{\nu}`` is the modified Bessel function of the
second kind, ``\nu > 0`` and ``\theta > 0``.
"""
struct MaternKernel{T<:AbstractFloat} <: MercerKernel{T}
    nu::HyperParameter{T}
    rho::HyperParameter{T}
    MaternKernel{T}(ν::Real, ρ::Real) where {T<:AbstractFloat}  = new{T}(
        HyperParameter(convert(T,ν), interval(OpenBound(zero(T)), nothing)),
        HyperParameter(convert(T,ρ), interval(OpenBound(zero(T)), nothing))
    )
end
function MaternKernel(ν::T1=1.0, ρ::T2=one(T1)) where {T1<:Real,T2<:Real}
    MaternKernel{floattype(T1,T2)}(ν,ρ)
end

@inline function maternkernel(z::T, ν::T, ρ::T) where {T}
    v1 = sqrt(2ν) * z / ρ
    v1 = v1 < eps(T) ? eps(T) : v1  # Overflow risk as z -> Inf
    2 * (v1/2)^(ν) * besselk(ν, v1) / gamma(ν)
end

@inline basefunction(::MaternKernel) = SquaredEuclidean()
@inline function kappa(κ::MaternKernel{T}, z::T) where {T}
    maternkernel(z, getvalue(κ.nu), getvalue(κ.rho))
end


using PyPlot
using Optim

function get_ln_delta(N, d_VC, epsilon)
    return log(4) + d_VC * log(2N) -(epsilon^2)*N/8
end;

d_VC = 10;
epsilon = 0.05;
N_arr = [400000. 420000. 440000. 460000. 480000.];
for N in N_arr
    delta = exp(get_ln_delta(N, d_VC, epsilon));
    println("N = $N, delta = $delta")
end

function get_error(N, d_VC, epsilon, target_delta)
    delta = exp(get_ln_delta(N, d_VC, epsilon));
    return (delta - target_delta)^2;
end;

result = optimize(N -> get_error(N, 10, 0.05, 0.05), 400000, 500000)

N = result.minimum;
err = result.f_minimum;
if result.converged == true
    println("Solution:  N = $N  (error = $err)")
end

function get_vc_bound(N,d_VC,delta)
    inner = log(4) + d_VC * log(2N) - log(delta);
    return sqrt(8inner/N);
end;

function get_rademacher_penalty(N,d_VC,delta)
    inner = 2(log(2N) + d_VC * log(N))/N
    return sqrt(inner) + sqrt(-2log(delta)/N) + 1/N;
end;

function get_parrondo_and_van_den_broek(N,d_VC,delta)
    pvdb_err(epsilon,N,d_VC,delta) = (epsilon - sqrt((2epsilon + log(6) + d_VC*log(2N) - log(delta))/N))^2;
    result = optimize(epsilon -> pvdb_err(epsilon,N,d_VC,delta), 0.0, 10.0);
    if result.converged==true
        return result.minimum
    else
        return NaN
    end;
end;

function get_devroye(N,d_VC,delta)
    devroye_err(epsilon,N,d_VC,delta) = (epsilon - sqrt((4epsilon*(1+epsilon) + log(4) + d_VC*log(N^2) - log(delta))/(2N)))^2;
    result = optimize(epsilon -> devroye_err(epsilon,N,d_VC,delta), 0.0, 20.0);
    if result.converged==true
        return result.minimum
    else
        return NaN
    end;
end;

d_VC = 50;
delta = 0.05;

#N_arr = logspace(0,4,11);  #from 1 to 10k
N_arr = [5, 10, 50, 100, 500, 1000, 5000, 10000];
epsilon_vc = [ get_vc_bound(N,d_VC,delta) for N in N_arr ];
epsilon_rp = [ get_rademacher_penalty(N,d_VC,delta) for N in N_arr ];
epsilon_pv = [ get_parrondo_and_van_den_broek(N,d_VC,delta) for N in N_arr ];
epsilon_dv = [ get_devroye(N,d_VC,delta) for N in N_arr ];

function plot_epsilons(N_arr,epsilon_arr,label_arr)
    fig = figure(figsize=(7,4))
    ax = fig[:add_subplot](1,1,1)
    for i in 1:size(label_arr)[1]
        #ax[:loglog](N_arr,epsilon_arr[:,i],label=label_arr[i]);
        loglog(N_arr,epsilon_arr[:,i],"x-",label=label_arr[i]);
    end;
    #ax[:legend](loc="best",frameon=false);
    legend(loc="best",frameon=false,fontsize=10)
    grid(true);
end;

epsilon_arr = [ epsilon_vc epsilon_rp epsilon_pv epsilon_dv ];
label_arr = [ "VC", "Rademacher", "Parrondo & Van den Broek", "Devroye" ];
plot_epsilons(N_arr,epsilon_arr,label_arr);

println("N = $(N_arr[end])")
println("ϵ (VC)                       = $(epsilon_vc[end])")
println("ϵ (Rademacher)               = $(epsilon_rp[end])")
println("ϵ (Parrondo & Van den Broek) = $(epsilon_pv[end])")
println("ϵ (Devroye)                  = $(epsilon_dv[end])")

println("N = $(N_arr[1])")
println("ϵ (VC)                       = $(epsilon_vc[1])")
println("ϵ (Rademacher)               = $(epsilon_rp[1])")
println("ϵ (Parrondo & Van den Broek) = $(epsilon_pv[1])")
println("ϵ (Devroye)                  = $(epsilon_dv[1])")

function generate_dataset(;n_points=200)
    x = linspace(-1,1,n_points);
    y = sin(π*x);
    x_sample = rand(2);
    y_sample = sin(π*x_sample);
    return x,y,x_sample,y_sample;
end;

function fit_a(x_sample,y_sample)
    sum = x_sample[1]*y_sample[1]+x_sample[2]*y_sample[2]
    return sum/(x_sample[1]^2+x_sample[2]^2);
end;

function predict_y_a(x,a)
    return a .* x;
end;

function plot_xy(x,y,x_sample,y_sample,y_pred)
    figure(figsize=(7,5));
    plot(x,y,"m-",label=L"$\sin(\pi x)$");
    plot(x,y_pred,"g--",label="predicted");
    plot(x_sample,y_sample,"bo",label="sample data");
    grid(true);
    legend(loc="best",frameon=false);
end;
x,y,x_sample,y_sample = generate_dataset();
y_pred = predict_y_a(x,fit_a(x_sample,y_sample));
plot_xy(x,y,x_sample,y_sample,y_pred);

function get_avg_a(n_trials; n_points=200)
    a_arr = Float64[];
    for i in 1:n_trials
        x_sample = rand(2);
        y_sample = sin(π*x_sample);
        a = fit_a(x_sample,y_sample);
        push!(a_arr,a);
    end
    return mean(a_arr);
end;
a = get_avg_a(1e6);    #1.423, 1.425, 1.424, 1.427
println("<g(x)> = $(round(a,2))x");

function get_bias_variance(avg_a, n_trials; n_points=200)
    bias  = Float64[];
    var   = Float64[];
    for i in 1:n_trials
        x_sample = rand(2);
        y_sample = sin(π*x_sample);
        a = fit_a(x_sample,y_sample);
        g_bar_x = avg_a*x_sample;
        push!(bias,mean((g_bar_x - y_sample).^2));
        push!(var,mean((g_bar_x - a .* x_sample).^2));
    end
    return bias, var;
end;

bias,var = get_bias_variance(a, 1e7);
println("<bias> = $(mean(bias))");
println("<var>  = $(mean(var))");


"""
    GaussianCopula{d,MT}

Fields:
  - Σ::MT - covariance matrix

Constructor

    GaussianCopula(Σ)

The [Gaussian Copula](https://en.wikipedia.org/wiki/Copula_(probability_theory)#Gaussian_copula) is the
copula of a [Multivariate normal distribution](http://en.wikipedia.org/wiki/Multivariate_normal_distribution). It is constructed as :

```math
C(\\mathbf{x}; \\boldsymbol{\\Sigma}) = F_{\\Sigma}(F_{\\Sigma,i}^{-1}(x_i),i\\in 1,...d)
```
where ``F_{\\Sigma}`` is a cdf of a gaussina random vector and `F_{\\Sigma,i}` is the ith marignal cdf, while ```\\Sigma`` is the covariance matrix.

It can be constructed in Julia via:
```julia
C = GaussianCopula(Σ)
```

The random number generation works as expected:
```julia
rand(C,1000)
# or
Random.rand!(C,u)
```

And yo can fit the distribution via :
```julia
fit(GaussianCopula,data)
```
"""
struct GaussianCopula{d,MT} <: EllipticalCopula{d,MT}
    Σ::MT
    function GaussianCopula(Σ)
        make_cor!(Σ)
        return new{size(Σ,1),typeof(Σ)}(Σ)
    end
end
U(::Type{T}) where T<: GaussianCopula = Distributions.Normal
N(::Type{T}) where T<: GaussianCopula = Distributions.MvNormal
function Distributions.fit(::Type{CT},u) where {CT<:GaussianCopula}
    dd = Distributions.fit(N(CT), quantile.(U(CT)(),u))
    Σ = Matrix(dd.Σ)
    return GaussianCopula(Σ)
end


function bisection(f′, a, b, ϵ)
    if a > b; a, b = b, a; end
    ya, yb = f′(a), f′(b)
    if ya == 0; b = a; end
    if yb == 0; a = b; end
    while b-a > ϵ
        x = (a+b)/2;
        y = f′(x);
        if y == 0
            a, b = x, x;
        elseif sign(y) == sign(ya)
            a = x;
        else
            b = x;
        end
    end
    return (a, b);
end


function blkdiag(blocks::Array{Array{Pol{T}, 2}, 1}) where {T<:Number}
    m = sum(size.(blocks, 1))
    n = sum(size.(blocks, 2))
    J = zeros(Pol{T}, m, n)
    offset1 = 0
    offset2 = 0
    for blk in blocks
        J[offset1+1:offset1+size(blk, 1), offset2+1:offset2+size(blk, 2)] = blk
        offset1 += size(blk, 1)
        offset2 += size(blk, 2)
    end
    return J
end

"""
creates the matrix block of size n×(n+1) corresponding to column index n
"""
function colBlock(n::Int)
    var = Pol([0, 1])
    J = zeros(Pol{Int}, n, n+1)
    for ii in 1:n
        J[ii, ii] = var
        J[ii, ii+1] = -1
    end
    return J
end

colBlock(arr::AbstractArray{Int, 1}) = blkdiag(colBlock.(arr))

"""
creates the matrix block of size (n+1)×n corresponding to row index n
"""
function rowBlock(n::Int)
    var = Pol([0, 1])
    J = zeros(Pol{Int}, n+1, n)
    @inbounds for ii in 1:n
        J[ii, ii] = -1
        J[ii+1, ii] = var
    end
    return J
end

rowBlock(arr::AbstractArray{Int, 1}) = blkdiag(rowBlock.(arr))

"""
creates the block matrix corresponding to infinite eigenvalue of multiplicity n
"""
function infBlock(n::Int)
    var = Pol([0, 1])
    J = zeros(Pol{Int}, n, n)
    @inbounds for ii in 1:n-1
        J[ii, ii] = -1
        J[ii+1, ii] = var
    end
    J[n, n] = -1
    return J
end

infBlock(arr::AbstractArray{Int, 1}) = blkdiag(infBlock.(arr))

"""
creates the block matrix of size n×n corresponding to eigenvalue λ
"""
function eigBlock(λ::T, n::Int) where {T<:Number}
    var = Pol([0, 1])
    J = zeros(Pol{T}, n, n)
    @inbounds for ii in 1:n-1
        J[ii, ii] = var - λ
        J[ii+1, ii] = -1
    end
    J[n, n] = var - λ
    return J
end

eigBlock(λ::Number, arr::AbstractArray{Int, 1}) = blkdiag(eigBlock.(λ, arr))


"""
    SEIso <: Isotropic{SqEuclidean}

Isotropic Squared Exponential kernel (covariance)
```math
k(x,x') = σ²\\exp(- (x - x')ᵀ(x - x')/(2ℓ²))
```
with length scale ``ℓ`` and signal standard deviation ``σ``.
"""
mutable struct SEIso{T<:Real} <: Isotropic{SqEuclidean}
    "Squared length scale"
    ℓ2::T
    "Signal variance"
    σ2::T
    "Priors for kernel parameters"
    priors::Array
end

"""
Squared Exponential kernel function

    SEIso(ll::T, lσ::T)

# Arguments:
- `ll::Real`: length scale (given on log scale)
- `lσ::Real`: signal standard deviation (given on log scale)
"""
SEIso(ll::T, lσ::T) where T = SEIso{T}(exp(2 * ll), exp(2 * lσ), [])

function set_params!(se::SEIso, hyp::AbstractVector)
    length(hyp) == 2 || throw(ArgumentError("Squared exponential has two parameters, received $(length(hyp))."))
    se.ℓ2, se.σ2 = exp(2 * hyp[1]), exp(2 * hyp[2])
end

get_params(se::SEIso{T}) where T = T[log(se.ℓ2) / 2, log(se.σ2) / 2]
get_param_names(se::SEIso) = [:ll, :lσ]
num_params(se::SEIso) = 2

cov(se::SEIso, r::Number) = se.σ2*exp(-0.5*r/se.ℓ2)

@inline dk_dll(se::SEIso, r::Real) = r/se.ℓ2*cov(se,r)
@inline function dk_dθp(se::SEIso, r::Real, p::Int)
    if p==1
        return dk_dll(se, r)
    elseif p==2
        return dk_dlσ(se, r)
    else
        return NaN
    end
end


export frac_iso, is_frac_iso

"""
    frac_iso(G,H)

Find a doubly stochastic matrix `S` so that `AS=SB` where `A` and `B`
are the adjacency matrices of `G` and `H` respectively.
"""
function frac_iso(G::SimpleGraph, H::SimpleGraph)
    if deg(G) != deg(H) || degdeg(G) != degdeg(H) # quick basic check before LP
        error("The graphs are not fractionally isomorphic")
    end
    A = adjacency(G)
    B = adjacency(H)
    return frac_iso(A, B)
end

function frac_iso(A::Matrix, B::Matrix)
    n, c = size(A)
    nn, cc = size(B)
    @assert n == c && nn == cc "Matrices must be square"
    @assert n == nn "Matrices must have the same size"

    m = Model(get_solver())

    @variable(m, S[1:n, 1:n])

    for i = 1:n
        for k = 1:n
            @constraint(
                m,
                sum(A[i, j] * S[j, k] for j = 1:n) == sum(S[i, j] * B[j, k] for j = 1:n)
            )
        end
        @constraint(m, sum(S[i, j] for j = 1:n) == 1)
        @constraint(m, sum(S[j, i] for j = 1:n) == 1)
    end

    for i = 1:n
        for j = 1:n
            @constraint(m, S[i, j] >= 0)
        end
    end

    optimize!(m)
    status = Int(termination_status(m))

    if status != 1
        error("The graphs are not fractionally isomorphic")
    end

    return value.(S)

end

"""
    is_frac_iso(G,H)
Test if two graphs are fractionally isomorphic.
"""
function is_frac_iso(G::SimpleGraph, H::SimpleGraph)::Bool
    try
        f = frac_iso(G, H)
        return true
    catch
        return false
    end
    false
end


using EngEconomics, Roots, Plots, ColorSchemes
plotly()

# Given
constructionYears = 5 # Construction takes this many years
constructionStarts = 5 # Construction starts in this many years
constructionA = 20000 # Construction annual cost in $1000, so if this number is 20,000, the real price is 20,000,000
maintenanceStarts = 1 + constructionYears + constructionStarts # Number of years when maintenance for the project will begin. It is simply the sum of how long construction will take and when construction starts plus 1 year
maintenanceA = 2000 # Maintenance annual cost in $1000, so if this number is 2,000, then the real price is 2,000,000
inflationRate = 0.03 # Inflation rate
MARR = 0.07 # Project's MARR
projectLifeTime = 30 # I did this in my head (starts working in 2020, supposed to last till 2050)

# Find
# (a.) The present worth of the project
realMARR = (1 + MARR) / (1 + inflationRate) - 1
NPW = (-constructionA * seriesPresentAmountFactor(realMARR, constructionYears)) * presentWorthFactor(realMARR, constructionStarts) + (-maintenanceA * seriesPresentAmountFactor(realMARR, projectLifeTime)) * presentWorthFactor(realMARR, constructionYears + constructionStarts)

# (b.) Construct a sensitivity graph showing the effects of 5 percent and 10
#      percent increases and decreases in the construction costs, maintenance
#      costs and inflation rate. To which is the present worth most sensitive?
sensitivity = [0.9, 0.95, 1.0, 1.05, 1.1]

# Construction Costs
realMARR = (1 + MARR) / (1 + inflationRate) - 1
NPW_constructionCost = (-constructionA * sensitivity * seriesPresentAmountFactor(realMARR, constructionYears)) * presentWorthFactor(realMARR, constructionStarts) .+ (-maintenanceA * seriesPresentAmountFactor(realMARR, projectLifeTime)) * presentWorthFactor(realMARR, constructionYears + constructionStarts)
plot(sensitivity, NPW_constructionCost)

# Maintenance Costs
realMARR = (1 + MARR) / (1 + inflationRate) - 1
NPW_maintenanceCost = (-constructionA * seriesPresentAmountFactor(realMARR, constructionYears)) * presentWorthFactor(realMARR, constructionStarts) .+ (-maintenanceA * sensitivity * seriesPresentAmountFactor(realMARR, projectLifeTime)) * presentWorthFactor(realMARR, constructionYears + constructionStarts)
plot!(sensitivity, NPW_maintenanceCost)

# Inflation Rate
realMARR_inflationRate = (1 + MARR) ./ (1 .+ sensitivity * inflationRate) .- 1
NPW_inflationRate = (-constructionA * seriesPresentAmountFactor.(realMARR_inflationRate, constructionYears)) .* presentWorthFactor.(realMARR_inflationRate, constructionStarts) .+ (-maintenanceA * seriesPresentAmountFactor.(realMARR_inflationRate, projectLifeTime)) .* presentWorthFactor.(realMARR_inflationRate, constructionYears + constructionStarts)
plot!(sensitivity, NPW_inflationRate)

println("From the plot, it\'s clear that the NPW cost is most sensitive to the initial construction cost because the slope of that line is much bigger than that of the other lines")


#=
Please see
https://computationalmindset.com/en/neural-networks/ordinary-differential-equation-solvers.html#ode1
for details
=#

#=Original problem
x′ + x = sin t + 3 cos 2t
x(0) = 0
=#

#=Esplicit form of equation
x′ = sin t  + 3 cos 2t − x
=#

#=Analytical solution
x(t) = 1/2 sin t - 1/2 cos t + 3/5 cos 2t + 6/5 sin 2t - 1/10 e^-t
=#

using DifferentialEquations
using Plots

ode_fn(x,p,t) = sin(t) + 3.0 * cos(2.0 * t) - x

an_sol(t) = (1.0/2.0) * sin(t) - (1.0/2.0) * cos(t) +
            (3.0/5.0) * cos(2.0*t) + (6.0/5.0) * sin(2.0*t) -
            (1.0/10.0) * exp(-t)

t_begin=0.0
t_end=10.0
tspan = (t_begin,t_end)
x_init=0.0

prob = ODEProblem(ode_fn, x_init, tspan)
num_sol = solve(prob, Tsit5(), reltol=1e-8, abstol=1e-8)

plot(num_sol.t, an_sol.(num_sol.t),
    linewidth=2, ls=:dash,
    title="ODE 1st order IVP solved by D.E. package",
    xaxis="t", yaxis="x",
    label="analytical",
    legend=true)
plot!(num_sol,
    linewidth=1,
    label="numerical")


"""
    PInterval <: SimpleProbabilitySet

Credal set of categorical distributions defined by lower and upper probabilities.
"""
struct PInterval <: SimpleProbabilitySet
    plower::Vector{Float64}
    pupper::Vector{Float64}
end

"""
    PInterval(p::Vector{Float64}, uncsize::Float64)

Probability intervals centered on 'p' of width ± 'uncsize' (with saturated bounds)
"""
function PInterval(p::Vector{Float64}, uncsize::Float64)
    pl = zeros(length(p))
    pu = ones(length(p))
    for (i, pi) in enumerate(p)
        pl[i] = max(0.0, pi - uncsize)
        pu[i] = min(1.0, pi + uncsize)
    end
    PInterval(pl, pu)
end

"""
    PInterval(p::Vector{Float64})

Degenerate probability interval of only pmf 'p'
"""
PInterval(p::Vector{Float64}) = PInterval(p,p) # degenerate PInterval

"""
    PInterval()

Default probability interval of p1 ∈ [0.2, 0.3], p2 ∈ [0.7, 0.8]
"""
PInterval() = PInterval([0.2, 0.7], [0.3, 0.8]) # default PInterval

"""
    psample(plower::Vector{Float64}, pupper::Vector{Float64})

Generate a probability distribution, uniformly at random, from the ambiguity set defined by the `pupper` and `plower` probabilities.
"""
function psample(plower::Vector{Float64}, pupper::Vector{Float64})
    n = length(plower)
    A = zeros(Float64, 2n+1, n)
    for i = 1:n
        A[2i - 1,i] = 1.0
        A[2i, i] = -1.0
    end
    A[end,:] = ones(n)
    bc = Array{Float64}(2n+1)
    for i = 1:n
        # due to limitations in hitandrun, set upper limit to 0+eps if upper limit is actually 0.0
        bc[2i - 1] = pupper[i] == 0.0 ? pupper[i] + 0.0000001 : pupper[i]
        bc[2i] = -1.0 * plower[i]
    end
    bc[end] = 1.0
    d = vcat(fill("<=", 2n),"=")
    nsample = 1
    rseed = rand(1:10_000)
    rout = R"""
    library(hitandrun)
    set.seed($rseed)
    constr <- list(constr = $A, rhs = $bc, dir = $d)
    samples <- hitandrun(constr, n.samples = $nsample, thin = ($n) ^ 3)
    """
    p = reshape(rcopy(rout), n)
end

psample(pint::PInterval) = psample(pint.plower, pint.pupper)


module problem126

export Cuboid

description = """
Cuboid layers

The minimum number of cubes to cover every visible face on a cuboid measuring 3 x 2 x 1 is twenty-two.
If we then add a second layer to this solid it would require forty-six cubes to cover every visible face,
the third layer would require seventy-eight cubes, and the fourth layer would require one-hundred and eighteen cubes to cover every visible face.

However, the first layer on a cuboid measuring 5 x 1 x 1 also requires twenty-two cubes;
similarly the first layer on cuboids measuring 5 x 3 x 1, 7 x 2 x 1, and 11 x 1 x 1 all contain forty-six cubes.

We shall define C(n) to represent the number of cuboids that contain n cubes in one of its layers.
So C(22) = 2, C(46) = 4, C(78) = 5, and C(118) = 8.
It turns out that 154 is the least value of n for which C(n) = 10.

Find the least value of n for which C(n) = 1000.
"""

# A cube with length, width and height
immutable Cuboid
  l :: Int
  w :: Int
  h :: Int
end

# calculates the middle section of the n'th layer
# equal to (the perimeter of original cube + diagonals) * height or original cube
middle(c :: Cuboid, n :: Int) = (2 * (c.l + c.w) + 4 * (n - 1)) * c.h

# Recurrance relation for calculating the top section of the n'th layer
# For n = 1, just the area of the top of original cube
# otherwise, equal to the top of the previous layer + a one cube thick rim from the middle of previous layer
top(c :: Cuboid, n :: Int) =
  if (n == 1)
    return (c.l * c.w)
  else
    return (top(c, n-1) + div(middle(c, n-1), c.h))
  end

# Calculate the n'th layer, top and bottom are symetric + middle section
layer(c :: Cuboid, n :: Int) = 2 * top(c, n) + middle(c, n)

# Calculate the size of all layers of cuboid up to a maximum
# re-uses the results of previous iterations to efficiently calculate the next
function layers(c :: Cuboid, maxsize :: Int)
  n = 1
  t = top(c, n)
  m = middle(c, n)
  x = 0
  result = Int[]
  while true
    x = 2 * t + m
    if x > maxsize break end
    push!(result, x)
    t = t + div(m, c.h)
    n += 1
    m = middle(c, n)
  end
  return result
end

# Search for a solution x, to C(x) == N, for x in 1:guess
# The lower bound is given by layer 1 of cuboid (Length, 1, 1)
# Once a solution is found < lower bound, search can stop
function search(N :: Int, guess :: Int)
  counts = zeros(Int, guess)
  l = 1
  lowerbound = layer(Cuboid(l, 1, 1), 1)
  while lowerbound <= guess
    for i = 1:lowerbound-1
      if counts[i] == N
        return i
      end
    end
    w = 1
    while w <= l && layer(Cuboid(l, w, 1), 1) <= guess
      h = 1
      while h <= w && layer(Cuboid(l, w, h), 1) <= guess
        sizes = layers(Cuboid(l,w,h), guess)
        counts[sizes] += 1
        h += 1
      end
      w += 1
    end
    l += 1
    lowerbound = layer(Cuboid(l, 1, 1), 1)
  end
  println("no solution found < $guess")
  return -1
end

using Base.Test

@test layer(Cuboid(3, 2, 1), 1) == 22
@test layer(Cuboid(3, 2, 1), 2) == 46
@test layer(Cuboid(3, 2, 1), 3) == 78
@test layer(Cuboid(3, 2, 1), 4) == 118
@test layer(Cuboid(5, 1, 1), 1) == 22
@test layer(Cuboid(5, 3, 1), 1) == 46
@test layer(Cuboid(7, 2, 1), 1) == 46
@test layer(Cuboid(11, 1, 1), 1) == 46
@test layers(Cuboid(3,2,1), 78) == [22, 46, 78]
@test search(10, 200) == 154

end


function atan2(y,x)
    if x>0
        return atan(y/x)
    elseif (x<0 && y>=0)
        return atan(y/x)+pi
    elseif (x<0 && y<0)
        return atan(y/x)-pi
    elseif (x==0 && y>0)
        return pi/2
    elseif (x==0 && y<0)
        return -pi/2
    else
        return NaN
    end
end


# Fixed point classification in 2D
using DrWatson
@quickactivate "NonlinearDynamicsTextbook"
include(srcdir("style.jl"))
using DynamicalSystems, PyPlot, Random

alleigs = []
fig, axs = subplots(2,3; figsize = (figx, 2figy))
ax = gca()
xgrid = -5:0.05:5
ygrid = xgrid
ux = zeros(length(xgrid), length(ygrid))
uy = copy(ux)

Ms = [
     [1.0  0.8;
      0.4 1.0],
     [-1.0  -0.8;
      1.8  -1.0],
     [-1.0  0;
      1  1],
     [0 -1;
      1  0],
]

titles = ["repulsive node", "attractive spiral", "hyperbolic/saddle", "center"]

using LinearAlgebra
function stream_eigs!(ax, M, c = "C0")
    for (i, x) in enumerate(xgrid)
        for (j, y) in enumerate(ygrid)
            ux[i, j], uy[i, j] = M * [x, y]
        end
    end

    ax.streamplot(Vector(xgrid), Vector(ygrid), ux', uy';
        linewidth = 1.5, density = 0.5, color = c, arrowsize = 2,
    )
    ev = eigen(M)
    push!(alleigs, ev.values)
    if eltype(ev.values) <: Float64
        e1 = ev.vectors[:, 1]
        e2 = ev.vectors[:, 2]
        for e in (e1, e2)
            ax.plot(e[1] .* 2xgrid, e[2] .* 2ygrid; color = "C2", ls = "dashed")
        end
    end
    ax.set_xlim(xgrid[1], xgrid[end])
    ax.set_ylim(ygrid[1], ygrid[end])
    ax.set_xticks([])
    ax.set_yticks([])
    ax.plot(0, 0; marker = "o", mec = "C2", mew = 1,
        markersize = 12, mfc = :white, zorder = 99
    )
end

function tovec(e)
    if eltype(e) <: Real
        e1 = e
        e2 = zeros(2)
    else
        e1 = [e[1].re, e[2].re]
        e2 = [e[1].im, e[2].im]
    end
    return e1, e2
end

for (i, M) in enumerate(Ms)
    ax = axs[i]
    color = i < 3 ? "C$(i-1)" : "C$(i)"
    stream_eigs!(ax, M, color)
    ax.set_title(titles[i]; color = color)
    # Plot eigenvalues
    e1, e2 = tovec(alleigs[i])
    axs[5].scatter(e1, e2; color = color, s = 200, zorder = 99)
end

# Set axis of eigenval plot
axs[5].spines["left"].set_position("center")
axs[5].spines["bottom"].set_position("center")
axs[5].grid(false)
axs[5].spines["right"].set_color("none")
axs[5].spines["top"].set_color("none")
axs[5].set_title("eigenvalues"; color = "k")
# axs[5].set_yticklabels([])
# axs[5].set_xticklabels([])
axs[5].set_xlim(-2, 2)
axs[5].set_ylim(-1.8, 1.8)

# Plot limit cycle
function vanderpoll(u, p, t)
    x, y = u;
    xdot = p*(x - x^3/3 - y)
    ydot = x/p
    return SVector(xdot, ydot)
end

ds = ContinuousDynamicalSystem(vanderpoll, [0.1, 0.2], 0.5)
tr = trajectory(ds, 7.0; Ttr = 100.0)
axs[6].clear()
axs[6].plot(columns(tr)...; color = "C2", linestyle = "--")
for (i, x) in enumerate(xgrid)
    for (j, y) in enumerate(ygrid)
        ux[i, j], uy[i, j] = ds.f(SVector(x, y), ds.p, 0)
    end
end
axs[6].streamplot(Vector(xgrid), Vector(ygrid), ux', uy';
    linewidth = 1.5, density = 0.5, color = "C5", arrowsize = 2
)
ax = axs[6]
ax.set_xlim(xgrid[1], xgrid[end])
ax.set_ylim(ygrid[1], ygrid[end])
ax.set_xticks([])
ax.set_yticks([])
ax.set_title("attractive limit cycle"; color = "C5")

fig.subplots_adjust(bottom = 0.02, left = 0.02, top = 0.92, right = 0.97, hspace = 0.2)
wsave(plotsdir("2", "2ddynamics"), fig)


@doc raw"""
    PowerKernel([γ=1])

The Power Kernel is a negative definite kernel given by:

```math
\kappa(\mathbf{x},\mathbf{y}) =
\|\mathbf{x} - \mathbf{y} \|^{2\gamma}
\qquad \gamma \in (0,1]
```
"""
struct PowerKernel{T<:AbstractFloat} <: NegativeDefiniteKernel{T}
    gamma::HyperParameter{T}
    PowerKernel{T}(γ::Real) where {T<:AbstractFloat} = new{T}(
        HyperParameter(convert(T,γ), interval(OpenBound(zero(T)), ClosedBound(one(T))))
    )
end
PowerKernel(γ::T1 = 1.0) where {T1<:Real} = PowerKernel{floattype(T1)}(γ)

@inline powerkernel(z::T, γ::T) where {T<:AbstractFloat} = z^γ

@inline basefunction(::PowerKernel) = SquaredEuclidean()
@inline kappa(κ::PowerKernel{T}, z::T) where {T} = powerkernel(z, getvalue(κ.gamma))


include("getdigit_n.jl")
include("myprintf.jl")
using Printf
using .GetDigit_n
using .MyPrintf

const DIGIT = 100
const MAXN = 500

function gaulegofpi(n)
    aₙ = BigFloat(1)
    bₙ = BigFloat(1) / sqrt(BigFloat(2))
    tₙ = parse(BigFloat, "0.25")
    pₙ = BigFloat(1)

    for i = 1:n
        aₙ₊₁ = (aₙ + bₙ) / BigFloat(2)
        bₙ = sqrt(aₙ * bₙ)
        tₙ = tₙ - pₙ * (aₙ - aₙ₊₁) * (aₙ - aₙ₊₁)
        pₙ = BigFloat(2) * pₙ
        aₙ = aₙ₊₁
    end

    return (aₙ + bₙ) * (aₙ + bₙ) / (BigFloat(4) * tₙ)
end

function main()
    n = GetDigit_n.getn(gaulegofpi, MAXN, DIGIT, true)
    precision = GetDigit_n.getdigit(gaulegofpi, n, DIGIT, true)
    precision = GetDigit_n.getdigit_2(gaulegofpi, n, DIGIT, precision)

    setprecision(precision)
    MyPrintf.myprintf("π（計算値） = ", DIGIT, gaulegofpi(n))
    MyPrintf.myprintf("π（厳密値） = ", DIGIT, BigFloat(π))
    @printf("n = %d, 計算%d桁で、%d桁求まりました\n", n, precision, DIGIT)
end

main()


"""
`assemble_system!` - assemble the matrix for the left hand side of the problem
"""
function assemble_system!(problem::RandomODEProblem)
    # construct the first row of the discrete Laplacian
    problem.A[1,1] = -2
    problem.A[1,2] = 1

    # construct middle rows
    for j ∈ 2:problem.nₓ - 1
        problem.A[j, j- 1] = 1
        problem.A[j, j] = -2
        problem.A[j, j + 1] = 1
    end

    # construct last row
    problem.A[problem.nₓ, problem.nₓ - 1] = 1
    problem.A[problem.nₓ, problem.nₓ] = -2

    # adjust for random potential
    problem.A = spdiagm(problem.V) - problem.A/problem.Δx^2

    return problem
end


using Pkg
Pkg.activate(".")
using PlotlyJS, Zygote, LinearAlgebra
using Dash, DashCoreComponents, DashHtmlComponents
using DashBootstrapComponents

function frenet_frame(x, y, z)
    ẋ(t) = gradient(x, t)[1]
    ẏ(t) = gradient(y, t)[1]
    ż(t) = gradient(z, t)[1]

    ẍ(t) = hessian(x, t)
    ÿ(t) = hessian(y, t)
    z̈(t) = hessian(z, t)

    α̇(t) = [ẋ(t), ẏ(t), ż(t)]
    α̈(t) = [ẍ(t), ÿ(t), z̈(t)]
    function out(t)
        T = α̇(t)
        T = T / norm(T)
        ad = α̈(t)
        N = ad - dot(ad, T)T
        N = N / norm(N)
        B = cross(T, N)
        T, N, B
    end
end

function uv_surface(uv_domain, ϕ; uv_res = [40, 40])
    us = LinRange(uv_domain[1], uv_domain[2], uv_res[1])
    vs = LinRange(uv_domain[3], uv_domain[4], uv_res[2])
    u_res, v_res = uv_res
    xs = zeros(u_res, v_res)
    ys = zeros(u_res, v_res)
    zs = zeros(u_res, v_res)
    for (i, u) in enumerate(us)
        for (j, v) in enumerate(vs)
            xs[i, j], ys[i, j], zs[i, j] = ϕ(u, v)
        end
    end
    surface(x = xs, y = ys, z = zs, showscale = false)
end

function xyz(ps)
    x = [p[1] for p in ps]
    y = [p[2] for p in ps]
    z = [p[3] for p in ps]
    return x, y, z
end

function torus_knot(m::Int64, n::Int64; R = 2, r = 1, uv_res = [220, 220])
    ϕ(u, v) = [(R + r * cos(u)) * cos(v), (R + r * cos(u)) * sin(v), r * sin(u)]
    α(t) = ϕ(m * t, n * t)
    x(t) = α(t)[1]
    y(t) = α(t)[2]
    z(t) = α(t)[3]

    F = frenet_frame(x, y, z)
    function ψ(u, v)
        T, N, B = F(u)
        ρ = r / 3
        α(u) + ρ * cos(v) * N + ρ * sin(v) * B
    end
    uv_surface([-π, 1.02π, -π, π], ψ, uv_res = uv_res)
end
description = dcc_markdown(raw"Torus knots are obtained by winding a curve around a torus
 ($m$ times in parallel direction and $n$ times in the meridian direction). More concretely, if

 $$\varphi(u,v) = ((R+r\cos(u))\cos(v), (R+r\cos(u))\sin(v), r\sin(u))$$

 is the torus parametrization, the curve

 $$\alpha(t) = \varphi(mt,nt)\text{ for }0\leq t \leq 2\pi $$

is said to be knot of the type $(m,n)$. These knots can be made into surfaces by thickening them using the Frenet frame over $\alpha$
 ")
sd = 4.0
scene3d = Dict(
    :xaxis => Dict(:range => [-sd, sd], :autorange => false),
    :yaxis => Dict(:range => [-sd, sd], :autorange => false),
    :zaxis => Dict(:range => [-sd, sd], :autorange => false),
    :aspectratio => Dict(:x => 1, :y => 1, :z => 1),
)

layout = Layout(scene = scene3d, width = 600, height = 600)

mathjax = "https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.4/MathJax.js?config=TeX-MML-AM_CHTML"
app = dash(external_scripts = [mathjax], external_stylesheets = [dbc_themes.BOOTSTRAP])
app.title = "Dash Torus Knots"

knot = torus_knot(2, 3)
graph = dbc_spinner(dcc_graph(id = "graph", figure = plot(knot, layout)))

slider_m = dcc_slider(
    id = "slider_m",
    min = 1,
    max = 7,
    step = 1,
    value = 3,
    marks = Dict([i => ("m=$(i)") for i = 1:10]),
)
slider_n = dcc_slider(
    id = "slider_n",
    min = 1,
    max = 7,
    step = 1,
    value = 2,
    marks = Dict([i => ("n=$(i)") for i = 1:10]),
)

input = dbc_row([dbc_col(slider_m, width = Dict("size" => 3, "order" => "first", "offset" => 3)),
 dbc_col(slider_n, width = Dict("size" => 3, "order" => "last", "offset" => 0))])
app.layout = dbc_container(
    [
        html_h1(html_center("Torus Knots"))
        html_br()
        description
        html_center(graph)
        html_center(input)
        html_br()
        html_br()
    ],
)

callback!(
    app,
    Output("graph", "figure"),
    Input("slider_m", "value"),
    Input("slider_n", "value"),
) do m, n
    knot = torus_knot(m, n)
    return plot(knot, layout)
end
run_server(app, "0.0.0.0", debug = true)


function gcd(x,y)
    if x == 0
        return y
    else
        return gcd(y%x,x)
    end
end
function find(arr,n)
    r = arr[1]
    for i=2:n
        r = gcd(arr[i],r)
    end
    return r
end
v = [8 16 8 4]
len = length(v)
println("The GCD of the array is = ",find(v,len))


"""
    Fibonacci{T} <: UnivariateAlgorithm

The Fibonacci search algorithm optimize a univariate function by iteratively shrinking the
initial bracket around the minimum. The ratio is computed from the Fibonacci sequence.
"""
struct Fibonacci{T} <: UnivariateAlgorithm
    ϵ::T

    function Fibonacci{T}(ϵ) where {T}
        0 < ϵ < 1 ? new{T}(ϵ) : error("`ϵ` must be in the interval (0, 1)")
    end
end

"""
    Fibonacci([ϵ=0.01])

Initialize Fibonacci search algorithm with optional argument `ϵ` in the interval (0, 1)
to control how close the last iterate is to the previous one. Refer to Kochenderfer and
Wheeler's "Algorithms for Optimization" for more information.
"""
Fibonacci(ϵ=0.01) = Fibonacci{typeof(ϵ)}(ϵ)

function _optimize(f, lower::T, upper::T, alg::Fibonacci;
        reltol, abstol, maxiter) where {T}
    ϵ = T(alg.ϵ)
    s = (1 - √5) / (1 + √5)
    p = 1 / (φ * (1 - s^(maxiter + 1)) / (1 - s^maxiter))
    x = T(p * lower + (1 - p) * upper)
    yx = f(x)
    converged = false
    iter = 1
    while true
        x_tol = reltol * abs(x) + abstol
        if abs(upper - lower) < 2x_tol
            converged = true
            break
        end
        iter == maxiter && break
        iter += 1
        if iter == maxiter
            new_x = T(ϵ * upper + (1 - ϵ) * x)
        else
            new_x = T(p * upper + (1 - p) * lower)
        end
        new_yx = f(new_x)
        if yx > new_yx
            lower, x, yx = x, new_x, new_yx
        else
            lower, upper = new_x, lower
        end
        p = 1 / (φ * (1 - s^(maxiter - iter + 2)) / (1 - s^(maxiter - iter + 1)))
    end
    return Solution(converged, iter, x, yx)
end


export l1_distance, l2_distance, squared_l2_distance, linf_distance, lp_distance

""" l1_distance computes the Manhattan's distance """
function l1_distance(a::AbstractVector{T}, b::AbstractVector{T})::Float64 where {T <: Real}
    d::T = zero(T)

    @fastmath @inbounds @simd for i = 1:length(a)
	    m = a[i] - b[i]
        d += ifelse(m > 0, m, -m)
    end

    return d
end

""" l2_distance computes the Euclidean's distance """
function l2_distance(a::AbstractVector{T}, b::AbstractVector{T})::Float64 where {T <: Real}
    d::T = zero(T)

    @fastmath @inbounds @simd for i = 1:length(a)
        m = a[i] - b[i]
        d += m * m
    end

    return sqrt(d)
end

""" squared_l2_distance computes the Euclidean's distance but squared """
function squared_l2_distance(a::AbstractVector{T}, b::AbstractVector{T})::Float64 where {T <: Real}
    d::T = zero(T)

    @fastmath @inbounds @simd for i = 1:length(a)
        m = a[i] - b[i]
        d += m * m
    end

    return d
end

""" linf_distance computes the max distance """
function linf_distance(a::AbstractVector{T}, b::AbstractVector{T})::Float64 where {T <: Real}
   d::T = zero(T)

    @fastmath @inbounds @simd for i = 1:length(a)
        m = abs(a[i] - b[i])
        d = max(d, m)
    end

    return d
end

"""
lp_distance creates a function that computes computes generic Minkowski's distance
"""
function lp_distance(p_::Real)
    p::Float64 = convert(Float64, p_)
    invp = 1.0 / p

    function _lp(a::AbstractVector{T}, b::AbstractVector{T})::Float64 where T
        d::T = zero(T)

        @fastmath @inbounds @simd for i = 1:length(a)
            m = abs(a[i] - b[i])
            d += m ^ p
        end

        d ^ invp
    end
end


# # Chebyshev center
# Boyd & Vandenberghe, "Convex Optimization"
# Joëlle Skaf - 08/16/05
#
# Adapted for Convex.jl by Karanveer Mohan and David Zeng - 26/05/14
#
# The goal is to find the largest Euclidean ball (i.e. its center and
# radius) that lies in a polyhedron described by affine inequalites in this
# fashion: $P = \{x : a_i'*x \leq b_i, i=1,\ldots,m \}$ where $x \in \mathbb{R}^2$.

using Convex, LinearAlgebra, SCS

# Generate the input data
a1 = [ 2;  1];
a2 = [ 2; -1];
a3 = [-1;  2];
a4 = [-1; -2];
b = ones(4, 1);

# Create and solve the model
r = Variable(1)
x_c = Variable(2)
p = maximize(r)
p.constraints += a1' * x_c + r * norm(a1, 2) <= b[1];
p.constraints += a2' * x_c + r * norm(a2, 2) <= b[2];
p.constraints += a3' * x_c + r * norm(a3, 2) <= b[3];
p.constraints += a4' * x_c + r * norm(a4, 2) <= b[4];
solve!(p, () -> SCS.Optimizer(verbose=0))
p.optval

# Generate the figure
x = range(-1.5, stop=1.5, length=100);
theta = 0:pi/100:2*pi;
using Plots
plot(x, x -> -x * a1[1] / a1[2] + b[1] / a1[2])
plot!(x, x -> -x * a2[1]/ a2[2] + b[2] / a2[2])
plot!(x, x -> -x * a3[1]/ a3[2] + b[3] / a3[2])
plot!(x, x -> -x * a4[1]/ a4[2] + b[4] / a4[2])
plot!(evaluate(x_c)[1] .+ evaluate(r) * cos.(theta), evaluate(x_c)[2] .+ evaluate(r) * sin.(theta), linewidth = 2)
plot!(title ="Largest Euclidean ball lying in a 2D polyhedron", legend = nothing)


## This example comes from https://diffeqflux.sciml.ai/dev/Flux/

using ComponentArrays
using OrdinaryDiffEq
using Plots
using UnPack

using DiffEqFlux: sciml_train
using Flux: glorot_uniform, ADAM, σ, relu
using Optim: LBFGS

struct MeasurementNoise{T}
    sigma::T
end
Base.:(+)(array, mn::MeasurementNoise) = array .+ randn(size(array)...).*mn.sigma

# Problem setup
u0 = Float32[2.; 0.]
datasize = 50
tspan = (0.0f0, 7f0)
tspan2 = (0.0f0, 25f0)

# Make truth data
function trueODEfunc(du, u, p, t)
    true_A = [-0.1 2.0; -2.0 -0.1]
    du .= ((u.^3)'true_A)'
end

t = range(tspan[1], tspan[2], length = datasize)
# t = Float32.(vcat(range(0.0, 0.9, length=10), 10 .^ range(log10(tspan[1]+1), log10(tspan[2]), length=datasize-10)))
t = Float32.([0; 10 .^ range(log10(tspan[1] + 0.01), log10(tspan[2]), length=datasize-1)])
prob = ODEProblem(trueODEfunc, u0, tspan2)
ode_sol = solve(prob, Tsit5())
ode_data = Array(ode_sol(t)) + MeasurementNoise(0.1)

# Function for creating neural layer components
neural_layer(in, out) = ComponentArray{Float32}(W=glorot_uniform(out, in), b=zeros(out))

# Dense neural layer function
dense(layer, activation=identity) = u -> activation.(layer.W * u + layer.b)

# Neural ODE function
dudt(u, p, t) = u.^3 |> dense(p.L1, σ) |> dense(p.L2)

prob = ODEProblem(dudt, u0, tspan2)

# Create optimization parameter vector
n_hidden = 50
layers = (L1=neural_layer(2, n_hidden), L2=neural_layer(n_hidden, 2))
θ = ComponentArray(u=u0, p=layers)

# Prediction and loss functions
predict_n_ode(θ) = Array(solve(prob, Tsit5(), u0=θ.u, p=θ.p, saveat=t))
full_sol(θ) = solve(prob, Tsit5(), u0=θ.u, p=θ.p)

function loss_n_ode(θ)
    pred = predict_n_ode(θ)

    loss = sum(abs2, ode_data .- pred)/datasize + 0.1*(sum(abs, θ.p)/length(θ.p))
    return loss, pred
end
loss_n_ode(θ)

# anim = Animation()

 # Callback function to observe training
cb = function (θ, loss, pred; doplot=false)
    display(loss)
    # plot current prediction against data
    train_sol = full_sol(θ)

    pl_1 = plot(train_sol, vars=1)
    plot!(pl_1, ode_sol, vars=1)
    scatter!(pl_1, t, pred[1,:])
    scatter!(pl_1, t, ode_data[1,:], legend=false)

    pl_2 = plot(train_sol, vars=2)
    plot!(pl_2, ode_sol, vars=2)
    scatter!(pl_2, t, pred[2,:])
    scatter!(pl_2, t, ode_data[2,:], legend=false)

    pl_3 = plot(train_sol, vars=(1,2), label = "prediction")
    plot!(pl_3, ode_sol, vars=(1,2), label = "truth")
    scatter!(pl_3, pred[1,:], pred[2,:], label = "predicted data")
    scatter!(pl_3, ode_data[1,:], ode_data[2,:], label = "measured data")
    plot!(pl_3, hcat(pred[1,:], ode_data[1,:])', hcat(pred[2,:], ode_data[2,:])', label = false, color=:lightgray, legend=:bottomright)

    display(plot(plot(pl_1, pl_2, layout=(2,1), size=(400,500)), pl_3, layout=(1,2), size=(950,500)))
    # frame(anim)
    return false
end

# Display the ODE with the initial parameter values.
cb(θ, loss_n_ode(θ)...)

data = Iterators.repeated((), 1000)

res1 = sciml_train(loss_n_ode, θ, ADAM(0.05); maxiters=100, save_best=true, cb=cb)
cb(res1.minimizer, loss_n_ode(res1.minimizer)...; doplot=true)

res2 = sciml_train(loss_n_ode, res1.minimizer, LBFGS(), cb=cb)
cb(res2.minimizer, loss_n_ode(res2.minimizer)...; doplot=true)

# gif(anim, "DiffEqFlux.gif", fps=15)


# # Maximum a Posteriori

using Distributions
using HMMBase
using PyPlot
using Seaborn

rc("axes", xmargin = 0) # hide
set_style("whitegrid")  # hide

# Let's consider a simple time series with one outlier:
y = rand(1000)
y[100] = 10000
figure(figsize = (9,2)) # hide
plot(y)
gcf() # hide
#-

# An MLE approach for the observations distributions parameters
# may fail with a singularity (variance = 0) if an outlier
# becomes the only observation associated to some state:
hmm = HMM([0.9 0.1; 0.1 0.9], [Normal(0,1), Normal(5,1)])
try
    fit_mle(hmm, y, display=:iter);
catch e
    println(e)
end

# We can avoid this by putting a prior on the variance:
import ConjugatePriors: InverseGamma, NormalKnownMu, posterior_canon
import StatsBase: Weights

function fit_map(::Type{<:Normal}, observations, responsibilities)
    μ = mean(observations, Weights(responsibilities))

    ss = suffstats(NormalKnownMu(μ), observations, responsibilities)
    prior = InverseGamma(2,1)
    posterior = posterior_canon(prior, ss)
    σ2 = mode(posterior)

    Normal(μ, sqrt(σ2))
end

hmm, hist = fit_mle(hmm, y, estimator=fit_map, display=:iter)
figure(figsize = (4,3)) # hide
plot(hist.logtots)
xlabel("Iteration") # hide
ylabel("Log-likelihood") # hide
gcf() # hide
#-

hmm.B


"""
Cumulative Distribution Function of a Standard Gaussian Random Variable

		y=normcdf(x)

Where:\n
		x         = point of evaluation.

		y      = probability that a standard gaussian random variable is below x.

# Example
```julia-repl
julia> normcdf(0.0)
0.5
```
"""
function normcdf(x::number) where {number <: Number}
	return (1.0+erf(x/sqrt(2.0)))/2.0;
end

"""
Probability Distribution Function of a Standard Gaussian Random Variable

		y=normpdf(x)

Where:\n
		x         = point of evaluation.

		y      = value.

# Example
```julia-repl
julia> normpdf(0.0)
0.3989422804014327
```
"""
function normpdf(x::number) where {number <: Number}
	return exp(-0.5*x.^2)/sqrt(2*pi);
end

"""
Black & Scholes Price for European Options

		Price=blsprice(S0,K,r,T,σ,d=0.0,FlagIsCall=true)

Where:\n
		S0         = Value of the Underlying.
		K          = Strike Price of the Option.
		r          = Zero Rate.
		T          = Time to Maturity of the Option.
		σ          = Implied Volatility
		d          = Implied Dividend of the Underlying.
		FlagIsCall = true for Call Options, false for Put Options.

		Price      = price of the European Option.

# Example
```julia-repl
julia> blsprice(10.0,10.0,0.01,2.0,0.2,0.01)
1.1023600107733191
```
"""
function blsprice(S0::num1,K::num2,r::num3,T::num4,σ::num5,d::num6=0.0,FlagIsCall::Bool=true) where {num1 <: Number ,num2 <: Number ,num3 <: Number ,num4 <: Number ,num5 <: Number ,num6 <: Number}
	blscheck(S0,K,r,T,σ,d);
	d1=(log(S0/K)+(r-d+σ*σ*0.5)*T)/(σ*sqrt(T));
	d2=d1-σ*sqrt(T);
	Price=0.0;
	if FlagIsCall
	Price=S0*exp(-d*T)*normcdf(d1)-K*exp(-r*T)*normcdf(d2);
	else
	Price=-S0*exp(-d*T)*normcdf(-d1)+K*exp(-r*T)*normcdf(-d2);
	end
return Price;
end

"""
Black Price for European Options

		Price=blkprice(F0,K,r,T,σ,FlagIsCall=true)

Where:\n
		F0         = Value of the Forward.
		K          = Strike Price of the Option.
		r          = Zero Rate.
		T          = Time to Maturity of the Option.
		σ          = Implied Volatility.
		FlagIsCall = true for Call Options, false for Put Options.

		Price      = price of the European Option.

# Example
```julia-repl
julia> blkprice(10.0,10.0,0.01,2.0,0.2)
1.1023600107733191
```
"""
function blkprice(F0::num1,K::num2,r::num3,T::num4,σ::num5,FlagIsCall::Bool=true) where {num1 <: Number ,num2 <: Number ,num3 <: Number ,num4 <: Number ,num5 <: Number}
	blscheck(F0,K,r,T,σ);
	Price=blsprice(F0,K,r,T,σ,r,FlagIsCall);
return Price;
end

"""
Black & Scholes Delta for European Options

		Δ=blsdelta(S0,K,r,T,σ,d=0.0,FlagIsCall=true)

Where:\n
		S0         = Value of the Underlying.
		K          = Strike Price of the Option.
		r          = Zero Rate.
		T          = Time to Maturity of the Option.
		σ          = Implied Volatility
		d          = Implied Dividend of the Underlying.
		FlagIsCall = true for Call Options, false for Put Options.

		Δ          = delta of the European Option.

# Example
```julia-repl
julia> blsdelta(10.0,10.0,0.01,2.0,0.2,0.01)
0.5452173371920436
```
"""
function blsdelta(S0::num1,K::num2,r::num3,T::num4,σ::num5,d::num6=0.0,FlagIsCall::Bool=true) where {num1 <: Number ,num2 <: Number ,num3 <: Number ,num4 <: Number ,num5 <: Number ,num6 <: Number}
	blscheck(S0,K,r,T,σ,d);
	d1=(log(S0/K)+(r-d+σ*σ*0.5)*T)/(σ*sqrt(T));
	Δ=0.0;
	if FlagIsCall
	Δ=exp(-d*T)*normcdf(d1);
	else
	Δ=-exp(-d*T)*normcdf(-d1);
	end
return Δ;
end

"""
Black & Scholes Gamma for European Options

		Γ=blsgamma(S0,K,r,T,σ,d=0.0,FlagIsCall=true)

Where:\n
		S0         = Value of the Underlying.
		K          = Strike Price of the Option.
		r          = Zero Rate.
		T          = Time to Maturity of the Option.
		σ          = Implied Volatility
		d          = Implied Dividend of the Underlying.
		FlagIsCall = true for Call Options, false for Put Options.

		Γ          = gamma of the European Option.

# Example
```julia-repl
julia> blsgamma(10.0,10.0,0.01,2.0,0.2,0.01)
0.13687881535712826
```
"""
function blsgamma(S0::num1,K::num2,r::num3,T::num4,σ::num5,d::num6=0.0,FlagIsCall::Bool=true) where {num1 <: Number ,num2 <: Number ,num3 <: Number ,num4 <: Number ,num5 <: Number ,num6 <: Number}
	blscheck(S0,K,r,T,σ,d);
	d1=(log(S0/K)+(r-d+σ*σ*0.5)*T)/(σ*sqrt(T));
	Γ=exp(-d*T)*normpdf(d1)/(S0*σ*sqrt(T));
return Γ;
end

"""
Black & Scholes Vega for European Options

		ν=blsvega(S0,K,r,T,σ,d=0.0,FlagIsCall=true)

Where:\n
		S0         = Value of the Underlying.
		K          = Strike Price of the Option.
		r          = Zero Rate.
		T          = Time to Maturity of the Option.
		σ          = Implied Volatility
		d          = Implied Dividend of the Underlying.
		FlagIsCall = true for Call Options, false for Put Options.

		ν          = vega of the European Option.

# Example
```julia-repl
julia> blsvega(10.0,10.0,0.01,2.0,0.2,0.01)
5.475152614285131
```
"""
function blsvega(S0::num1,K::num2,r::num3,T::num4,σ::num5,d::num6=0.0,FlagIsCall::Bool=true) where {num1 <: Number ,num2 <: Number ,num3 <: Number ,num4 <: Number ,num5 <: Number ,num6 <: Number}
	blscheck(S0,K,r,T,σ,d);
	d1=(log(S0/K)+(r-d+σ*σ*0.5)*T)/(σ*sqrt(T));
	ν=S0*exp(-d*T)*normpdf(d1)*sqrt(T);
return ν;
end

"""
Black & Scholes Rho for European Options

		ρ=blsrho(S0,K,r,T,σ,d=0.0,FlagIsCall=true)

Where:\n
		S0         = Value of the Underlying.
		K          = Strike Price of the Option.
		r          = Zero Rate.
		T          = Time to Maturity of the Option.
		σ          = Implied Volatility
		d          = Implied Dividend of the Underlying.
		FlagIsCall = true for Call Options, false for Put Options.

		ρ          = rho of the European Option.

# Example
```julia-repl
julia> blsrho(10.0,10.0,0.01,2.0,0.2,0.01)
8.699626722294234
```
"""
function blsrho(S0::num1,K::num2,r::num3,T::num4,σ::num5,d::num6=0.0,FlagIsCall::Bool=true) where {num1 <: Number ,num2 <: Number ,num3 <: Number ,num4 <: Number ,num5 <: Number ,num6 <: Number}
	blscheck(S0,K,r,T,σ,d);
	d2=(log(S0/K)+(r-d-σ*σ*0.5)*T)/(σ*sqrt(T));
	if FlagIsCall
	ρ=K*exp(-r*T)*normcdf(d2)*T;
	else
		ρ=-K*exp(-r*T)*normcdf(-d2)*T;
	end
return ρ;
end

"""
Black & Scholes Theta for European Options

		Θ=blstheta(S0,K,r,T,σ,d=0.0,FlagIsCall=true)

Where:\n
		S0         = Value of the Underlying.
		K          = Strike Price of the Option.
		r          = Zero Rate.
		T          = Time to Maturity of the Option.
		σ          = Implied Volatility
		d          = Implied Dividend of the Underlying.
		FlagIsCall = true for Call Options, false for Put Options.

		Θ          = theta of the European Option.

# Example
```julia-repl
julia> blstheta(10.0,10.0,0.01,2.0,0.2,0.01)
-0.26273403060652334
```
"""
function blstheta(S0::num1,K::num2,r::num3,T::num4,σ::num5,d::num6=0.0,FlagIsCall::Bool=true) where {num1 <: Number ,num2 <: Number ,num3 <: Number ,num4 <: Number ,num5 <: Number ,num6 <: Number}
	blscheck(S0,K,r,T,σ,d);
	sqrtT       = sqrt(T);
	σ_sqrtT = σ .* sqrtT;

	d1 = (log(S0 ./ K) + (r - d + σ.^2 / 2) .* T)./ σ_sqrtT;

	shift = -exp(-d .* T) .* S0 .* normpdf(d1) .* σ / 2 ./ sqrtT;
	t1    = r .* K   .* exp(-r .* T);
	t2    = d .* S0 .* exp(-d .* T);
	Θ=0.0;
	if FlagIsCall
		Θ=shift - t1 .*      normcdf(d1 - σ_sqrtT)  + t2 .*  normcdf(d1)     ;
	else
		Θ=shift + t1 .* (1 - normcdf(d1 - σ_sqrtT)) + t2 .* (normcdf(d1) - 1);
	end
return Θ;
end

"""
Black & Scholes Lambda for European Options

		Λ=blslambda(S0,K,r,T,σ,d=0.0,FlagIsCall=true)

Where:\n
		S0         = Value of the Underlying.
		K          = Strike Price of the Option.
		r          = Zero Rate.
		T          = Time to Maturity of the Option.
		σ          = Implied Volatility
		d          = Implied Dividend of the Underlying.
		FlagIsCall = true for Call Options, false for Put Options.

		Λ          = lambda of the European Option.

# Example
```julia-repl
julia> blslambda(10.0,10.0,0.01,2.0,0.2,0.01)
4.945909973725978
```
"""
function blslambda(S0::num1,K::num2,r::num3,T::num4,σ::num5,d::num6=0.0,FlagIsCall::Bool=true) where {num1 <: Number ,num2 <: Number ,num3 <: Number ,num4 <: Number ,num5 <: Number ,num6 <: Number}
	blscheck(S0,K,r,T,σ,d);
	Price=blsprice(S0,K,r,T,σ,d,FlagIsCall);
	Δ=blsdelta(S0,K,r,T,σ,d,FlagIsCall);
	Λ=Δ*S0/Price;
return Λ;
end

"Check input for Black Scholes Formula"
function blscheck(S0::num1,K::num2,r::num3,T::num4,σ::num5,d::num6=0.0) where {num1 <: Number ,num2 <: Number ,num3 <: Number ,num4 <: Number ,num5 <: Number ,num6 <: Number}
lesseq(x::Complex,y::Complex) = real(x) <= real(y)
lesseq(x,y) = x <= y
if (lesseq(S0,zero(num1)))
	error("Spot Price Cannot Be Negative")
elseif (lesseq(K,zero(num2)))
	error("Strike Price Cannot Be Negative")
elseif (lesseq(T,zero(num4)))
	error("Time to Maturity Cannot Be Negative")
elseif (lesseq(σ,zero(num5)))
	error("Volatility Cannot Be Negative")
end
return;
end

"Brent Method: Scalar Equation Solver"
function brentMethod(f::Function, x0::Number, x1::Number,xtol::AbstractFloat=1e-14, ytol::AbstractFloat=1e-15)
		if xtol<0.0
		  throw(ErrorException("x tollerance cannot be negative"));
		end
	if ytol<0.0
		  throw(ErrorException("y tollerance cannot be negative"));
		end
		EPS = eps(x0)
	maxiter=80;
		y0 = f(x0)
		y1 = f(x1)
		if abs(y0) < abs(y1)
		    # Swap lower and upper bounds.
		    x0, x1 = x1, x0
		    y0, y1 = y1, y0
		end
		x2 = x0
		y2 = y0
		x3 = x2
		bisection = true
		for _ in 1:maxiter
		    # x-tolerance.
		    if abs(x1-x0) < xtol
		        return x1
		    end

		    # Use inverse quadratic interpolation if f(x0)!=f(x1)!=f(x2)
		    # and linear interpolation (secant method) otherwise.
		    if abs(y0-y2) > ytol && abs(y1-y2) > ytol
		        x = x0*y1*y2/((y0-y1)*(y0-y2)) +
		            x1*y0*y2/((y1-y0)*(y1-y2)) +
		            x2*y0*y1/((y2-y0)*(y2-y1))
		    else
		        x = x1 - y1 * (x1-x0)/(y1-y0)
		    end

		    # Use bisection method if satisfies the conditions.
		    delta = abs(2EPS*abs(x1))
		    min1 = abs(x-x1)
		    min2 = abs(x1-x2)
		    min3 = abs(x2-x3)
		    if (x < (3x0+x1)/4 && x > x1) ||
		       (bisection && min1 >= min2/2) ||
		       (!bisection && min1 >= min3/2) ||
		       (bisection && min2 < delta) ||
		       (!bisection && min3 < delta)
		        x = (x0+x1)/2
		        bisection = true
		    else
		        bisection = false
		    end
		    y = f(x)
		    # y-tolerance.
		    if abs(y) < ytol
		        return x
		    end
		    x3 = x2
		    x2 = x1
		    if sign(y0) != sign(y)
		        x1 = x
		        y1 = y
		    else
		        x0 = x
		        y0 = y
		    end
		    if abs(y0) < abs(y1)
		        # Swap lower and upper bounds.
		        x0, x1 = x1, x0
		        y0, y1 = y1, y0
		    end
		end
		throw(ErrorException("Max iteration exceeded, possible wrong result"));
end

export blsimpv
"""
Black & Scholes Implied Volatility for European Options

		σ=blsimpv(S0,K,r,T,Price,d=0.0,FlagIsCall=true,xtol=1e-14,ytol=1e-15)

Where:\n
		S0         = Value of the Underlying.
		K          = Strike Price of the Option.
		r          = Zero Rate.
		T          = Time to Maturity of the Option.
		Price      = Price of the Option.
		d          = Implied Dividend of the Underlying.
		FlagIsCall = true for Call Options, false for Put Options.

		σ          = implied volatility of the European Option.

# Example
```julia-repl
julia> blsimpv(10.0,10.0,0.01,2.0,2.0)
0.3433730534290586
```
"""
function blsimpv(S0::num1,K::num2,r::num3,T::num4,Price::num5,d::num6=0.0,FlagIsCall::Bool=true,xtol::Real=1e-14,ytol::Real=1e-15) where {num1 <: Real ,num2 <: Real ,num3 <: Real ,num4 <: Real ,num5 <: Real ,num6 <: Real}
if (Price< num5(0))
	throw(ErrorException("Option Price Cannot Be Negative"));
end
blscheck(S0,K,r,T,0.1,d);
f(x)=(blsprice(S0,K,r,T,x,d,FlagIsCall)-Price);
σ=brentMethod(f,0.001,1.2,xtol,ytol);

return σ;
end

export blkimpv
"""
Black Implied Volatility for European Options

		σ=blkimpv(F0,K,r,T,Price,FlagIsCall=true,xtol=1e-14,ytol=1e-15)

Where:\n
		F0         = Value of the Forward.
		K          = Strike Price of the Option.
		r          = Zero Rate.
		T          = Time to Maturity of the Option.
		Price      = Price of the Option.
		FlagIsCall = true for Call Options, false for Put Options.

		σ          = implied volatility of the European Option.

# Example
```julia-repl
julia> blkimpv(10.0,10.0,0.01,2.0,2.0)
0.36568658096623635
```
"""
function blkimpv(F0::num1,K::num2,r::num3,T::num4,Price::num5,FlagIsCall::Bool=true,xtol::Real=1e-14,ytol::Real=1e-15) where {num1 <: Real ,num2 <: Real ,num3 <: Real ,num4 <: Real ,num5 <: Real}
	blscheck(F0,K,r,T,0.1);
	σ=blsimpv(F0,K,r,T,Price,r,FlagIsCall,xtol,ytol)
	return σ;
end

##	ADDITIONAL Function

"""
Black & Scholes Psi for European Options

		Ψ=blspsi(S0,K,r,T,σ,d=0.0,FlagIsCall=true)

Where:\n
		S0         = Value of the Underlying.
		K          = Strike Price of the Option.
		r          = Zero Rate.
		T          = Time to Maturity of the Option.
		σ          = Implied Volatility
		d          = Implied Dividend of the Underlying.
		FlagIsCall = true for Call Options, false for Put Options.

		Ψ          = psi of the European Option.

# Example
```julia-repl
julia> blspsi(10.0,10.0,0.01,2.0,0.2,0.01)
-10.904346743840872
```
"""
function blspsi(S0::num1,K::num2,r::num3,T::num4,σ::num5,d::num6=0.0,FlagIsCall::Bool=true) where {num1 <: Number ,num2 <: Number ,num3 <: Number ,num4 <: Number ,num5 <: Number ,num6 <: Number}
	blscheck(S0,K,r,T,σ,d);
	d1=(log(S0/K)+(r-d+σ*σ*0.5)*T)/(σ*sqrt(T));
	if FlagIsCall
	Ψ=-S0*exp(-d*T)*normcdf(d1)*T;
	else
		Ψ=S0*exp(-d*T)*normcdf(-d1)*T;
	end
return Ψ;
end

"""
Black & Scholes Vanna for European Options

		Vanna=blsvanna(S0,K,r,T,σ,d=0.0,FlagIsCall=true)

Where:\n
		S0         = Value of the Underlying.
		K          = Strike Price of the Option.
		r          = Zero Rate.
		T          = Time to Maturity of the Option.
		σ          = Implied Volatility
		d          = Implied Dividend of the Underlying.
		FlagIsCall = true for Call Options, false for Put Options.

		Vanna        = vanna of the European Option.

# Example
```julia-repl
julia> blsvanna(10.0,10.0,0.01,2.0,0.2,0.01)
0.2737576307142566
```
"""
function blsvanna(S0::num1,K::num2,r::num3,T::num4,σ::num5,d::num6=0.0,FlagIsCall::Bool=true) where {num1 <: Number ,num2 <: Number ,num3 <: Number ,num4 <: Number ,num5 <: Number ,num6 <: Number}
	blscheck(S0,K,r,T,σ,d);
	d1=(log(S0/K)+(r-d+σ*σ*0.5)*T)/(σ*sqrt(T));
	d2=d1-σ*sqrt(T);
	Vanna=-exp(-d*T)*normpdf(d1)*d2/σ;
return Vanna;
end


function  compute()
	numer=1
	denom=1

	for d=10:99
		for n =10:d-1
			n0=n%10
			n1=Int((n-n0)/10)
			d0=d%10
			d1=Int((d-d0)/10)
			if (n1==d0 && n0*d==n*d1)||(n0==d1&&n1*d==n*d0)
				numer*=n
				denom*=d
			end
		end
	end

	return Int(denom/gcd(numer,denom))
end

println(compute())


# # Introduciton to Lotka-Volterra Competition

using Parameters
using DifferentialEquations
using ForwardDiff
using Plots
pyplot();

# # Introduction/Overview
# This notebook is simply looking at the two species L-V competiti9on model. Here, we are exploring
# the geometry and dynamics of this model.
#
# # Model and Equilibria
# The model we will use is the classical LV competition model:

include("lv_model.jl")

# # Evaluationg the model -- time series

let
    u0 = [0.5, 0.5]
    t_span = (0.0, 100.0)
    p = LVPar()

    prob = ODEProblem(lv_comp, u0, t_span, p)
    sol = solve(prob, reltol = 1e-8)

    plot(sol)
    xlabel!("time")
    ylabel!("Density")
end

# # Equilibria and Isoclines
# From the above equations we can generate the Jacobian:

# Make a numerical (not symbolic version using the ForwardDiff.jl library.
# This is what you will want to do most of the time)
jac(u, p) = ForwardDiff.jacobian(u -> lv_comp(u, p, NaN), u)

# Here we have the jacobian evaluated at the point `u = [0.5, 0.5]`, with the parameter set `par`
jac([0.5, 0.5], LVPar())

# First solve for $f_1$ and $f_2$ to determine when functions are equal to 0 (i.e., when $du_1/dt$
# and $du_2/dt = 0$). These are known as the isoclines, or nullclines, and describe the set of
# solutions when $u_1$ and $u_2$ do not change.

include("lv_symbolic.jl")

# We want to solve our equations for $u_1$ so that we can plot the isoclines as a function of  $u_1$
# ($u_1$ on the y-axis), but here I am keeping all possible solutions, with respect to both $u_1$
# and $u_2$. You should be able to see that the first two solutions ($u_1 = K_1 - \alpha_{12}u_2$
# and $u_2 = (K_1 - u_1)/\alpha_{12}$) are equivalent.
#
# The solutions $u_1 = K_1 - \alpha_{12}u_2$ and $u_1 = (K_2 - u_2)/\alpha_{21}$ are the equations
# for our isoclines for $du_1/dt = 0$ and $du_2/dt = 0$, respectively. We can find the interior
# equilibrium where these lines intersect (i.e., both $u_1$ and $u_2$ do not change). In this case
# we get one interior equilibrium, but note that in more complicated models we can get more interior
# equilibria.

# # Now plot the isoclines
# We will again use the manipulate function to see how our parameters change the isoclines. This can
# immediately tell us a lot about our equilibria and stability.

function iso1(u2, p)
    @unpack α12, K1 = p
    return K1 - α12 * u2 / K1
end
#-
function iso2(u2, p)
    @unpack α21, K2 = p
    return (K2 - u2) / α21
end
#-
let
    p = LVPar()
    u2s = range(0, 5, length = 100)

    fig = plot(u2s, [iso1(u2, p) for u2 in u2s], label = "u1 = 0")
    plot!(u2s, [iso2(u2, p) for u2 in u2s], label = "u2 = 0")
    xlims!(0, 3)
    ylims!(0, 5)
    xlabel!("u2")
    ylabel!("u1")
end


module LinearShiftRegisters

export LSR_period, LCG_period

function _repeated_substring(source)
	len = length(source)

	# had to account for weird edge case in which length 2 vectors always returns themselves
	if len < 3
		len == 1 && return nothing

		s1, s2 = source
		if len == 2 && s1 == s2
			return s1
		else
			return nothing
		end
	end

	# Check candidate strings
	for i in 1:(len ÷ 2 + 1)
		repeat_count, remainder = divrem(len, i)

		# Check for no leftovers characters, and equality when repeated
		if remainder == 0 && source == repeat(source[1:i], repeat_count)
			return source[1:i]#, repeat_count
		end
	end

	return nothing
end

function _LSR_period(const_vec::AbstractVector, init_vec::AbstractVector)
	sequence_vec = reverse(init_vec)

	res = nothing
	while isnothing(res)
		new_val = mod(sum(map(i -> const_vec[i] * sequence_vec[end - (i - 1)], 1:4)), 2)
		push!(sequence_vec, new_val)
		res = _repeated_substring(sequence_vec)
	end

	return res
end

@doc raw"""
```julia
LSR_period(const_str::AbstractVector{T}, init_str::AbstractVector{R}) where {T, R <: Integer} -> AbstractVector
LSR_period(const_str::AbstractString, init_str::AbstractString) -> String
```

Given a starting binary string (`init_str`) and a constant mutating string (`const_str`), `LSR_period` will find the repeating sequence generated from these.

!!! note

    As this method looks for recurring sequences, it is possible that this function does not terminate—or maybe it does terminate, but I haven't waited long enough to find out.

---

### Description

A (linear) shift register with feedback is an arrangement of registers in a row, each register being capable of holding either the digit 1 (on) or 0 (off).  Suppose the system has ``n`` registers, ``R_1,\ldots,R_n``, and ``X_i(t)`` denotes the content of register ``R_i`` at time ``t``.  Initially, let the system be given

Given

### Examples

```julia
julia> LSR_period("1001", "0100")
"001000111101011"

julia> LSR_period("1001", "0101")
"101011001000111"
```
"""
function LSR_period(const_vec::AbstractVector{T}, init_vec::AbstractVector{R}) where {T, R <: Integer}
	return _LSR_period(const_vec, init_vec)
end
function LSR_period(const_str::AbstractString, init_str::AbstractString)
	const_vec = parse.(Int8, collect(const_str))
	init_vec = parse.(Int8, collect(init_str))

	return join(_LSR_period(const_vec, init_vec))
end

function _LCG_period(m::Integer, a::Integer, b::Integer, x0::T) where {T <: Integer}
	sequence_vec = T[x0]

	new_x, res = x0, nothing
	while isnothing(res)
		new_x =  mod(a * new_x + b, m)
		push!(sequence_vec, new_x)
		res = _repeated_substring(sequence_vec)
	end

	return res
end

@doc raw"""
```julia
LCG_period(m::Integer, a::Integer, b::Integer, x0::Integer) -> String
```

Given `m`, `a`, `b`, and `x0`, finds the period generated by these parameters from a Linear Congruential Generator, and returns this sequence as a string.

!!! note

    As this method looks for recurring sequences, it is possible that this function does not terminate—or maybe it does terminate, but I haven't waited long enough to find out.

---

### Description

Choose m, a positive integer.  Choose a, b (mod m) integers.

The Linear Congruential Generator is given to us by

```math
x_1 \equiv ax_0+b\mod m\\
x_2 \equiv ax_1+b\mod m\\
\vdots\\
x_{n + 1} \equiv ax_n + b \mod m\\
\vdots
```

Eventually, ``x_n = x_i`` for some ``n``, ``i`` (let's suppose ``n>i`` and ``i`` is not necessarily ``0``).  Then, clearly,

```math
x_{n+1} = x_{i+1},\ldots.
```

So

```math
x_i, x_{i+1},\dots, x_{n-1}
```

Will be repeated forever.

From a cryptography point of view, LCG is too vulnerable (although all pseudorandom number generators are vulnerable), but it is still useful for other perposes that require pseudorandom numbers.

For essentially a more sophisticated version of LCG in the sense that it is based on recurring linear relations, see `LSR_period`.

---

### Examples

```julia
julia> println(LinearShiftRegisters._LCG_period(9, 5, 2, 0))
[0, 2, 3, 8, 6, 5]

julia> LCG_period(9, 5, 2, 0)
"023865"
```
"""
function LCG_period(m::Integer, a::Integer, b::Integer, x0::Integer)
	return join(_LCG_period(m, a, b, x0))
end

end # end module


"""
lsnormal(A,b)

Solve a linear least squares problem by the normal equations.
Returns the minimizer of ||b-Ax||.
"""
function lsnormal(A,b)

N = A'*A;  z = A'*b;
R = cholesky(N).U
w = forwardsub(R',z)                   # solve R'z=c
x = backsub(R,w)                       # solve Rx=z

return x
end

"""
lsqrfact(A,b)

Solve a linear least squares problem by QR factorization. Returns
the minimizer of ||b-Ax||.
"""
function lsqrfact(A,b)

Q,R = qr(A)
c = Q'*b
x = backsub(R,c)

return x
end

"""
qrfact(A)

QR factorization by Householder reflections. Returns Q and R.
"""
function qrfact(A)

    m,n = size(A)
    Qt = Matrix(Diagonal(ones(m)))
    R = float(copy(A))
    for k in 1:n
      z = R[k:m,k]
      v = [ -sign(z[1])*norm(z) - z[1]; -z[2:end] ]
      nrmv = norm(v)
      if nrmv < eps() continue; end  # skip this iteration
      v = v / nrmv;                  # simplifies other formulas
      # Apply the reflection to each relevant column of A and Q
      for j in 1:n
        R[k:m,j] -= v*( 2*(v'*R[k:m,j]) )
      end
      for j in 1:m
        Qt[k:m,j] -= v*( 2*(v'*Qt[k:m,j]) )
      end
    end

    return Qt',triu(R)
end


# Runs a simple Metropolis-Hastings (ie MCMC) algoritm to simulate two
# jointly distributed random variuables with probability density
# p(x,y)=exp(-(x^2+y^2)/s^2)/consNorm, where s>0 and consNorm is a
# normalization constant.
#
# Author: H. Paul Keeler, 2019.
# Website: hpaulkeeler.com
# Repository: github.com/hpaulkeeler/posts

#Note: Need the .+ for adding a scalar to an array
#Also need . for sqrt, exp, cos, sin etc and assinging scalars to arrays

using Distributions #for random simulations
using PyPlot #for plotting
using StatsBase #for histograms etc
using Random
using LinearAlgebra
using HCubature #for numerical integration
#using LaTeXStrings #for LateX in labels/titles etc
PyPlot.close("all");  # close all PyPlot figures

#set random seed for reproducibility
#Random.seed!(1234)

# Simulation window parameters
xMin = -1;
xMax = 1;
yMin = -1;
yMax = 1;

numbSim = 10 ^ 5;  # number of random variables simulated
numbSteps = 25;  # number of steps for the Markov process
numbBins = 50;  # number of bins for histogram
sigma = 2;  # standard deviation for normal random steps

# probability density parameters
s = .5;  # scale parameter for distribution to be simulated

function fun_lambda(x,y)
    return (exp.(-(x.^2+y.^2)./s^2));
end

#normalization constant -- UNDER CONSTRUCTION
consNorm,errorCub=hcubature(x -> fun_lambda(x[1],x[2]), [xMin, yMin], [xMax, yMax]);

function fun_p(x,y)
    return((fun_lambda(x,y)./consNorm).*(x.>=xMin).*(y.>=yMin).*(x.<=xMax).*(y.<=yMax));
end
xRand=(xMax-xMin).*rand(numbSim).+xMin; #random initial values
yRand=(yMax-yMin).*rand(numbSim).+yMin; #random initial values

probCurrent=fun_p(xRand,yRand); #current transition probabilities
for jj=1:numbSteps
    zxRand= xRand.+sigma.*rand(Normal(),numbSim);#take a (normally distributed) random step
    zyRand= yRand.+sigma.*rand(Normal(),numbSim);#take a (normally distributed) random step

    # Conditional random step needs to be symmetric in x and y
    # For example: Z|x ~ N(x,1) (or Y=x+N(0,1)) with probability density
    # p(z|x)=e(-(z-x)^2/2)/sqrt(2*pi)
    probProposal = fun_p(zxRand, zyRand);  # proposed probability

    # acceptance rejection step
    booleAccept=rand(numbSim) .< probProposal./probCurrent;
    # update state of random walk/Marjov chain
    xRand[booleAccept] = zxRand[booleAccept];
    yRand[booleAccept] = zyRand[booleAccept];
    # update transition probabilities
    probCurrent[booleAccept] = probProposal[booleAccept];
end

#histogram section: empirical probability density
histXY=fit(Histogram, (xRand,yRand),nbins=numbBins); #find histogram data
histXY=normalize(histXY,mode=:pdf); #normalize histogram
binEdges=histXY.edges; #retrieve bin edges
xValues=(binEdges[1][2:end]+binEdges[1][1:end-1])./2; #mid-points of bins
yValues=(binEdges[2][2:end]+binEdges[2][1:end-1])./2; #mid-points of bins
p_Estimate=(histXY.weights)
#create a meshgrid
X=[xValues[ii] for ii=1:length(xValues), jj=1:length(yValues)];
Y=[yValues[jj] for ii=1:length(xValues), jj=1:length(yValues)];

#analytic solution of probability density
p_Exact = fun_p(X, Y);

# Plotting
# Plot empirical estimate
fig1 = PyPlot.figure();
PyPlot.rc("text", usetex=true);
PyPlot.rc("font", family="serif");
surf(X, Y, p_Estimate, cmap=PyPlot.cm.plasma);
PyPlot.xlabel("x");
PyPlot.ylabel("y");
PyPlot.title("p(x,y) Estimate");

# Plot exact expression
fig2 = PyPlot.figure();
PyPlot.rc("text", usetex=true);
PyPlot.rc("font", family="serif")
surf(X, Y, p_Exact, cmap=PyPlot.cm.plasma);
PyPlot.xlabel("x");
PyPlot.ylabel("y");
PyPlot.title("p(x,y) Exact Expression");

#println("The program has completed.")


# In this tutorial, we will learn
#    - How to solve a simple nonlinear PDE in Gridap
#    - How to define the weak residual and its Jacobian
#    - How to setup and use a nonlinear solver
#    - How to define new boundaries from a given discrete model
#
# ## Problem statement
#
# The goal of this tutorial is to solve a nonlinear PDE in Gridap. For the sake of simplicity, we consider the $p$-Laplacian equation as the model problem. Specifically, the PDE  we want to solve is: find the scalar-field $u$ such that
#
# ```math
# \left\lbrace
# \begin{aligned}
# -\nabla \cdot \left( |\nabla u|^{p-2} \ \nabla u \right) = f\ &\text{in}\ \Omega,\\
# u = 0 \ &\text{on} \ \Gamma_0,\\
# u = g \ &\text{on} \ \Gamma_g,\\
# \left( |\nabla u|^{p-2}\ \nabla u \right)\cdot n = 0 \ &\text{on} \ \Gamma_{\rm N},
# \end{aligned}
# \right.
# ```
# with $p>2$.
# The computational domain $\Omega$ is the one depicted in next figure, which is the same as in the first tutorial. However, we slightly change the boundary conditions here. We impose homogeneous Dirichlet and homogeneous Neumann boundary conditions on $\Gamma_0$ and $\Gamma_{\rm N}$  respectively, and in-homogeneous Dirichlet conditions on $\Gamma_g$. The Dirichlet boundaries $\Gamma_0$ and $\Gamma_g$ are defined as the closure of the green and blue surfaces in next figure respectively, whereas the Neumann boundary is $\Gamma_{\rm N}\doteq\partial\Omega \setminus (\Gamma_0\cup\Gamma_g)$. In this example, we consider the values $p=3$, $f=1$, and $g=2$.
#
# ![](../assets/p_laplacian/model.png)
#
# ## Numerical scheme
#
# We discretize the problem with conforming Lagrangian FE spaces. For this formulation, the nonlinear weak form reads: find $u\in U_g$ such that $[r(u)](v) = 0$ for all $v\in V_0$. As in previous tutorials, the space $U_g$ is the set of functions in $H^1(\Omega)$ that fulfill the Dirichlet boundary conditions, whereas $V_0$ is composed by functions in $H^1(\Omega)$ that vanish at the Dirichlet boundary. The weak residual $r(u)$ evaluated at a function  $u\in U_g$ is the linear form defined as
#
# ```math
# [r(u)](v) \doteq \int_\Omega \nabla v \cdot \left( |\nabla u|^{p-2}\ \nabla u \right) \ {\rm d}\Omega - \int_\Omega v\ f \ {\rm d}\Omega.
# ```
#
# In order to solve this nonlinear weak equation, we consider a Newton-Raphson method, which is associated with a linearization of the problem in an arbitrary direction $\delta u\in V_0$, namely $[r(u+\delta u)](v)\approx [r(u)](v) + [j(u)](\delta u,v)$. In previous formula,  $j(u)$ is the Jacobian evaluated at $u\in U_g$, which is the bilinear form
#
# ```math
# [j(u)](\delta u,v) = \int_\Omega \nabla v \cdot \left( |\nabla u|^{p-2}\ \nabla \delta u \right) \ {\rm d}\Omega + (p-2) \int_\Omega \nabla v \cdot \left(  |\nabla u|^{p-4} (\nabla u \cdot \nabla \delta u) \nabla u  \right) \ {\rm d}\Omega.
# ```
#
# Note that the solution of this nonlinear PDE with a Newton-Raphson method, will require to discretize both the residual $r$ and the Jacobian $j$. In Gridap, this is done by following an approach similar to the one already shown in previous tutorials for discretizing the bilinear and linear forms associated with a linear FE problem. The specific details are discussed now.
#
# ## Discrete model
#
# As in previous tutorials, the first step to solve the PDE is to load a discretization of the computational domain. In this case, we load the model from the same file as in the first tutorial

using Gridap
model = DiscreteModelFromFile("../models/model.json")

# As stated before, we want to impose Dirichlet boundary conditions on $\Gamma_0$ and $\Gamma_g$,  but none of these boundaries is identified in the model. E.g., you can easily see by writing the model in vtk format

writevtk(model,"model")

# and by opening the file `"model_0"` in Paraview that the boundary identified as `"sides"` only includes the vertices in the interior of $\Gamma_0$, but here we want to impose Dirichlet boundary conditions in the closure of $\Gamma_0$, i.e., also on the vertices on the contour of $\Gamma_0$. Fortunately, the objects on the contour of $\Gamma_0$ are identified  with the tag `"sides_c"` (see next figure). Thus, the Dirichlet boundary $\Gamma_0$ can be built as the union of the objects identified as `"sides"` and `"sides_c"`.
#
# ![](../assets/p_laplacian/sides_c.png)
#
# Gridap provides a convenient way to create new object identifiers (referred to as "tags") from existing ones. First, we need to extract from the model, the object that holds the information about the boundary identifiers (referred to as `FaceLabeling`):

labels = get_face_labeling(model)

# Then, we can add new identifiers (aka "tags") to it. In the next line, we create a new tag called `"diri0"` as the union of the objects identified as `"sides"` and `"sides_c"`, which is precisely what we need to represent the closure of the Dirichlet boundary $\Gamma_0$.

add_tag_from_tags!(labels,"diri0",["sides", "sides_c"])

# We follow the same approach to build a new identifier for the closure of the Dirichlet boundary $\Gamma_g$. In this case, the boundary is expressed as the union of the objects identified with the tags `"circle"`, `"circle_c"`, `"triangle"`, `"triangle_c"`, `"square"`, `"square_c"`. Thus, we create a new tag for  $\Gamma_g$, called `"dirig"` simply as follows:

add_tag_from_tags!(labels,"dirig",
  ["circle","circle_c", "triangle", "triangle_c", "square", "square_c"])

# ## FE Space
#
# Now, we can build the FE space by using the newly defined boundary tags.

reffe = ReferenceFE(lagrangian,Float64,1)
V0 = TestFESpace(model,reffe,conformity=:H1,labels=labels,dirichlet_tags=["diri0", "dirig"])

# The construction of this space is essentially the same as in the first tutorial (we build a continuous scalar-valued Lagrangian interpolation of first order). However, we also pass here the `labels` object (that contains the newly created boundary tags). From this FE space, we define the trial FE spaces

g = 1
Ug = TrialFESpace(V0,[0,g])

# ## Nonlinear FE problem
#
# At this point, we are ready to build the nonlinear FE problem. To this end, we need to define the weak residual and also its corresponding Jacobian. This is done following a similar procedure to the one considered in previous tutorials to define the bilinear and linear forms associated with linear FE problems. We first need to define the usual objects for numerical integration:

degree=2
Ω = Triangulation(model)
dΩ = Measure(Ω,degree)

#
# On the one hand, the weak residual is built as follows

using LinearAlgebra: norm
const p = 3
flux(∇u) = norm(∇u)^(p-2) * ∇u
f(x) = 1
res(u,v) = ∫( ∇(v)⊙(flux∘∇(u)) - v*f )*dΩ

# Function `res` is the one representing the integrand of the weak residual $[r(u)](v)$. The first argument of function `res` stands for the function $u\in U_g$, where the residual is evaluated, and the second argument stands for a generic test function $v\in V_0$.
#
# On the other hand,  we (optionally) implement a function `jac` representing the Jacobian.
dflux(∇du,∇u) = (p-2)*norm(∇u)^(p-4)*(∇u⊙∇du)*∇u+norm(∇u)^(p-2)*∇du
jac(u,du,v) = ∫( ∇(v)⊙(dflux∘(∇(du),∇(u))) )*dΩ

# The first argument of function `jac` stands for function $u\in U_g$, where the Jacobian is evaluated. The second argument is a test function $v\in V_0$, and the third argument represents an arbitrary direction $\delta u \in V_0$.
#
# We finally construct the nonlinear FE problem

op = FEOperator(res,jac,Ug,V0)

# Here, we have constructed an instance of `FEOperator`, which is the type that represents a general nonlinear FE problem in Gridap. The constructor takes the functions representing the weak residual and Jacobian, and the test and trial spaces. If only the function for the residual is provided, the Jacobian is computed internally with automatic differentiation.

#
# ## Nonlinear solver phase
#
# We have already built the nonlinear FE problem. Now, the remaining step is to solve it. In Gridap, nonlinear (and also linear) FE problems can be solved with instances of the type `FESolver`.
#
# We construct an instance of `FESolver` as follows:

using LineSearches: BackTracking
nls = NLSolver(
  show_trace=true, method=:newton, linesearch=BackTracking())
solver = FESolver(nls)

# Note that the `NLSolver` function used above internally calls the `nlsolve` function of the [NLsolve](https://github.com/JuliaNLSolvers/NLsolve.jl) package with the provided key-word arguments. Thus, one can use any of the nonlinear methods available via the function `nlsolve` to solve the nonlinear FE problem. Here, we have selected a Newton-Raphson method with a back-tracking line-search from the [LineSearches](https://github.com/JuliaNLSolvers/LineSearches.jl) package.
#
# We are finally in place to solve the nonlinear FE problem. The initial guess is a `FEFunction`, which we build from a vector of random (free) nodal values:

import Random
Random.seed!(1234)
x = rand(Float64,num_free_dofs(Ug))
uh0 = FEFunction(Ug,x)
uh, = solve!(uh0,solver,op)

# We finish this tutorial by writing the computed solution for visualization (see next figure).

writevtk(Ω,"results",cellfields=["uh"=>uh])

# ![](../assets/p_laplacian/sol-plap.png)
#


"""

```
sphericalshellvolume(i,step)
```

Computes the volume of the spherical shell defined within [(i-1)*step,i*step].

"""
function sphericalshellvolume(i, step)
    rmin = (i - 1) * step
    return (4 * pi / 3) * ((rmin + step)^3 - rmin^3)
end


"""
    X = spiraldata(N)

Generates data points on a 2D spiral returned as a N×2 matrix X.
"""
function spiraldata(N, seed=1)

    rg = MersenneTwister(seed)

    t = rand(rg, Uniform(3, 15), N)

    x1 = 0.04 * t .* sin.(t) .+ randn(rg, N)*0.01

    x2 = 0.04 * t .* cos.(t) .+ randn(rg, N)*0.01

    [x1 x2]

end


#using Elefridge
using PyPlot

nbits = 8
nmax = 7
mi,ma = 1e-20,1e20
logmin = log(mi)
logmax = log(ma)
dlog = (logmax-logmin)/(2^nbits-2)              # 2^n-1-1 as 0x00 is reserved for 0
quants = exp.(logmin .+ Array(0:nmax)*dlog)     # the first quantised numbers
half_quants = quants[1:end-1] + diff(quants)/2         # half-way in lin-space
halflog_quants = exp.(logmin .+ (Array(0:nmax-1).+.5)*dlog)    # half-way in log-space
dlin = quants[2]-quants[1]

A = Array(quants[1]:dlin/100:quants[end])

# round to nearest in log space
L1 = round.((log.(A).-logmin)/dlog).+1
A1 = exp.(logmin .+ (L1.-1)*dlog)

B = vcat(A,ma)
A0 = Array(LogQuant8Array(B))

# round to nearest in lin space
c = 1/2 - 1/dlog*log(mi*(exp(dlog)+1)/2)

L2 = round.(c.+log.(A)/dlog).+1
A2 = exp.(logmin .+ (L2.-1)*dlog)

## plot
pygui(true)
fig,ax = subplots(1,1)
ax.plot(A,A1,"k--",lw=2,label="Round-to-nearest in log-space")
ax.plot(A,A2,"C1--",lw=2,label="Round-to-nearest in lin-space")
ax.plot(B,A0,"C2.-")

ax.plot([quants[1],half_quants[1]],[quants[1],quants[1]],"C1",lw=3.5)
ax.plot([quants[1],halflog_quants[1]],[quants[1],quants[1]],"k",lw=1.5)
for i in 1:length(halflog_quants)-1
    x0 = half_quants[i]
    x1 = half_quants[i+1]
    y = quants[i+1]
    ax.plot([x0,x1],[y,y],"C1",lw=3.5)
    x0 = halflog_quants[i]
    x1 = halflog_quants[i+1]
    ax.plot([x0,x1],[y,y],"k",lw=1.5)
end
ax.plot([half_quants[end],quants[end]],[quants[end],quants[end]],"C1",lw=3.5)
ax.plot([halflog_quants[end],quants[end]],[quants[end],quants[end]],"k",lw=1.5)

for (i,x) in enumerate(halflog_quants)
    labl = i==1 ? "Halfway in log-space" : nothing
    ax.axvline(x,color="k",lw=1,label=labl)
end

for (i,x) in enumerate(half_quants)
    labl = i==1 ? "Halfway in lin-space" : nothing
    ax.axvline(x,color="C1",lw=1,label=labl)
end

ax.set_yticks(quants)
ax.set_yticklabels([latexstring("x_{$i}") for i in 1:length(quants)])
ax.set_xticks(quants)
ax.set_xticklabels([latexstring("x_{$i}") for i in 1:length(quants)])
ax.set_xlabel(L"$x$ at full precision")
ax.set_ylabel(L"quantized $x$")

ax.set_xlim(quants[1],quants[end])
ax.set_ylim(quants[1]-dlin/2,quants[end]+dlin)

ax.legend(loc=2,framealpha=1)
ax.set_title("Round-to-nearest for log quantization",loc="left")

tight_layout()