# # Closest Correlation Matrix
#
# We consider the problem of finding the closest correlation matrix $X$ to a given random matrix $C$.
# With closest correlation matrix we mean a positive semidefinite matrix with ones on the diagonal.
# The problem is given by:
# $$
# \begin{array}{ll} \text{minimize} &  \frac{1}{2}||X - C||_F^2\\
# \text{subject to} &  X_{ii} = 1, \quad i=1,\dots,n \\
#                   &  X \succeq 0.
# \end{array}
# $$
# Notice that we use `JuMP` to model the problem. `COSMO` is chosen as the backend solver. And COSMO-specific settings are passed using
# the `optimizer_with_attributes()` function.

using COSMO, JuMP, LinearAlgebra, SparseArrays, Test, Random

#-
rng = Random.MersenneTwister(12345);
## create a random test matrix C
n = 8;
C = -1 .+ rand(rng, n, n) .* 2;
c = vec(C);

# Define problem in `JuMP`:
q = -vec(C);
r = 0.5 * vec(C)' * vec(C);
m = JuMP.Model(optimizer_with_attributes(COSMO.Optimizer, "verbose" => true));
@variable(m, X[1:n, 1:n], PSD);
x = vec(X);
@objective(m, Min, 0.5 * x' * x  + q' * x + r);
for i = 1:n
  @constraint(m, X[i, i] == 1.);
end

# Solve the `JuMP` model with `COSMO` and query the solution `X_sol`:
status = JuMP.optimize!(m);
obj_val = JuMP.objective_value(m);
X_sol = JuMP.value.(X);

# Double check result against known solution:
known_opt_val = 12.5406
known_solution =  [
  1.0         0.732562   -0.319491   -0.359985   -0.287543   -0.15578     0.0264044  -0.271438;
  0.732562    1.0         0.0913246  -0.0386357   0.299199   -0.122733    0.126612   -0.187489;
 -0.319491    0.0913246   1.0        -0.0863377   0.432948    0.461783   -0.248641   -0.395299;
 -0.359985   -0.0386357  -0.0863377   1.0         0.503379    0.250601    0.141151    0.286088;
 -0.287543    0.299199    0.432948    0.503379    1.0        -0.0875199   0.137518    0.0262425;
 -0.15578    -0.122733    0.461783    0.250601   -0.0875199   1.0        -0.731556    0.0841783;
  0.0264044   0.126612   -0.248641    0.141151    0.137518   -0.731556    1.0        -0.436274;
 -0.271438   -0.187489   -0.395299    0.286088    0.0262425   0.0841783  -0.436274    1.0  ];
@test isapprox(obj_val, known_opt_val , atol=1e-3)

#-
@test norm(X_sol - known_solution, Inf) < 1e-3


"""
ksintegrateUnrolled: integrate kuramoto-sivashinsky equation (Julia)

      u_t = -u*u_x - u_xx - u_xxxx, domain x in [0,Lx], periodic BCs

 inputs
          u = initial condition (vector of u(x) values on uniform gridpoints))
         Lx = domain length
         dt = time step
         Nt = number of integration timesteps
      nsave = save every nsave-th time step

 outputs
          u = final state, vector of u(x, Nt*dt) at uniform x gridpoints

This implementation has one improvements over ksintegrateInplace.jl. It explicitly
unrolls all the vector equations into for loops. For some reason this works
better (runs faster) than the unrolled loops produced by the Julia compiler.
"""
function ksintegrateUnrolled(u, Lx, dt, Nt)
    u = (1+0im)*u                       # force u to be complex
    Nx = length(u)                      # number of gridpoints
    kx = vcat(0:Nx/2-1, 0:0, -Nx/2+1:-1)# integer wavenumbers: exp(2*pi*kx*x/L)
    alpha = 2*pi*kx/Lx                  # real wavenumbers:    exp(alpha*x)
    D = 1im*alpha                       # spectral D = d/dx operator
    L = alpha.^2 - alpha.^4             # spectral L = -D^2 - D^4 operator
    G = -0.5*D                          # spectral -1/2 D operator, to eval -u u_x = 1/2 d/dx u^2

    # convenience variables
    dt2  = dt/2
    dt32 = 3*dt/2
    A =  ones(Nx) + dt2*L
    B = (ones(Nx) - dt2*L).^(-1)

    # compute FFTW plans
    FFT! = plan_fft!(u, flags=FFTW.ESTIMATE)
    IFFT! = plan_ifft!(u, flags=FFTW.ESTIMATE)

    # compute nonlinear term Nn == N(u^n) = -u u_x and Nn1
    Nn  = G.*fft(u.^2) # Nn == -1/2 d/dx (u^2) = -u u_x, spectral
    Nn1 = copy(Nn)     # Nn1 == N(u^{n-1}). For first timestep, let Nn1 = Nn
    FFT!*u

    # timestepping loop
    for n = 0:Nt

        Nn1 .= Nn
        Nn .= u

        IFFT!*Nn # in-place FFT

        @inbounds for i = 1:length(Nn)
            @fastmath Nn[i] = Nn[i]*Nn[i]
        end

        FFT!*Nn

        @inbounds for i = 1:length(Nn)
            @fastmath Nn[i] = G[i]*Nn[i]
        end

        @inbounds for i = 1:length(u)
            @fastmath u[i] = B[i]* (A[i] * u[i] + dt32*Nn[i] - dt2*Nn1[i])
        end

    end

    IFFT!*u
    real(u)
end


using PyPlot

"""
    y = ℓ(c, j, x)

Computes `j`th Lagrange polynomial with respect to the centres `c[k]`.
Thus, if `1 ≤ k ≤ n` then `ℓ` has degree `n-1` and `1 ≤ j ≤ n`.

"""
function ℓ(c, j, x)
    n = length(c)
    @assert 1 ≤ j ≤ n
    y = 1.0
    for k = 1:j-1
        y *= ( x - c[k] ) / ( c[j] - c[k] )
    end
    for k = j+1:n
        y *= ( x - c[k] ) / ( c[j] - c[k] )
    end
    return y
end

function ℓ(c, j, x::AbstractArray)
    n = length(x)
    y = fill(1.0, n)
    for k = 1:n
        y[k] = ℓ(c, j, x[k])
    end
    return y
end

n = 6
x = range(-1; stop=1, length=201)
c = range(-1; stop=1, length=n)

figure(1)
for j = 1:n
    plot(x, ℓ(c, j, x))
end
for j = 1:n
    plot(c[j:j], [1.0], "ok", c[j:j], [0.0], "ok")
end
grid(true)
title("Lagrange polynomials of degree $(n-1)")


"""
    krbf(X, Y; gamma = 1)
Compute a Radial-Basis-Function (RBF) kernel Gram matrix.
* `X` : Data.
* `Y` : Data.
* `gamma` : Scale parameter.

Given matrices `X` (n, p) and `Y` (m, p), the function returns
the (n, m) Gram matrix K(X, Y) = Phi(X) * Phi(Y)'.

The RBF kernel between two vectors x and y is computed by
exp(-gamma * ||x - y||^2).

## References

Scholkopf, B., Smola, A.J., 2002. Learning with kernels: support vector machines,
regularization, optimization, and beyond, Adaptive computation and machine learning.
MIT Press, Cambridge, Mass.

"""
function krbf(X, Y; gamma = 1)
    exp.(-gamma * euclsq(X, Y))
end

"""
    kpol(X, Y; degree = 1, gamma = 1, coef0 = 0)
Compute a polynomial kernel Gram matrix.
* `X` : Data.
* `Y` : Data.
* `degree` : Degree of the polynom.
* `gamma` : Scale of the polynom.
* `coef0` : Offset of the polynom.

Given matrices `X` (n, p) and `Y` (m, p), the function returns
the (n, m) Gram matrix K(X, Y) = Phi(X) * Phi(Y)'.

The polynomial kernel between two vectors x and y is computed by
(gamma * (x' * y) + coef0)^degree.

## References

Scholkopf, B., Smola, A.J., 2002. Learning with kernels: support vector machines,
regularization, optimization, and beyond, Adaptive computation and machine learning.
MIT Press, Cambridge, Mass.

"""
function kpol(X, Y; degree = 1, gamma = 1, coef0 = 0)
    K = gamma * X * Y' .+ coef0
    if degree > 1
        zK = copy(K)
        @inbounds for i = 1:(degree - 1)
            K .= K .* zK
        end
    end
    K
end


# ---
# title: 118. Pascal's Triangle
# id: problem118
# author: zhwang
# date: 2022-01-17
# difficulty: Easy
# categories: Array
# link: <https://leetcode.com/problems/pascals-triangle/description/>
# hidden: true
# ---
#
# Given a non-negative integer  _numRows_ , generate the first _numRows_ of
# Pascal's triangle.
#
# ![](https://upload.wikimedia.org/wikipedia/commons/0/0d/PascalTriangleAnimated2.gif)
# In Pascal's triangle, each number is the sum of the two numbers directly above
# it.
#
# **Example:**
#
#
#
#     Input: 5
#     Output:
#     [
#          [1],
#         [1,1],
#        [1,2,1],
#       [1,3,3,1],
#      [1,4,6,4,1]
#     ]
#
#
#
## @lc code=start
using LeetCode

pascals_triangle(numRows::Int) = [[binomial(k, i) for i in 0:k] for k in 0:(numRows - 1)]
## @lc code=end


## --- Weighted mean of an array

    """
    ```julia
    (wx, wσ, mswd) = awmean(x, σ)
    ```
    Weighted mean, absent the geochonologist's MSWD correction to uncertainty.
    """
    function awmean(x, σ)
        n = length(x)

        sum_of_values = sum_of_weights = χ2 = 0.0
        @inbounds @simd for i=1:n
            sum_of_values += x[i] / (σ[i]*σ[i])
            sum_of_weights += 1 / (σ[i]*σ[i])
        end
        wx = sum_of_values / sum_of_weights

        @inbounds @simd for i=1:n
            χ2 += (x[i] - wx) * (x[i] - wx) / (σ[i] * σ[i])
        end
        mswd = χ2 / (n-1)
        wσ = sqrt(1.0 / sum_of_weights)
        return wx, wσ, mswd
    end
    # function awmean(x::Array{<:Number}, σ::Array{<:Number})
    #     n = length(x)
    #
    #     sum_of_values = sum_of_weights = χ2 = 0.0
    #     @turbo for i=1:n
    #         sum_of_values += x[i] / (σ[i]*σ[i])
    #         sum_of_weights += 1 / (σ[i]*σ[i])
    #     end
    #     wx = sum_of_values / sum_of_weights
    #
    #     @turbo for i=1:n
    #         χ2 += (x[i] - wx) * (x[i] - wx) / (σ[i] * σ[i])
    #     end
    #     mswd = χ2 / (n-1)
    #     wσ = sqrt(1.0 / sum_of_weights)
    #     return wx, wσ, mswd
    # end
    export awmean

    """
    ```julia
    (wx, wσ, mswd) = gwmean(x, σ)
    ```
    Geochronologist's weighted mean, with "MSWD correction" to uncertainty,
    i.e., wσ is increased by a factor of sqrt(mswd)
    """
    function gwmean(x, σ)
        n = length(x)

        sum_of_values = sum_of_weights = χ2 = 0.0
        @inbounds @simd for i=1:n
            sum_of_values += x[i] / (σ[i]*σ[i])
            sum_of_weights += 1 / (σ[i]*σ[i])
        end
        wx = sum_of_values / sum_of_weights

        @inbounds @simd for i=1:n
            χ2 += (x[i] - wx) * (x[i] - wx) / (σ[i] * σ[i])
        end
        mswd = χ2 / (n-1)
        wσ = sqrt(mswd / sum_of_weights)
        return wx, wσ, mswd
    end
    # function gwmean(x::Array{<:Number}, σ::Array{<:Number})
    #     n = length(x)
    #     sum_of_values = sum_of_weights = χ2 = 0.0
    #     @turbo for i=1:n
    #         sum_of_values += x[i] / (σ[i]*σ[i])
    #         sum_of_weights += 1 / (σ[i]*σ[i])
    #     end
    #     wx = sum_of_values / sum_of_weights
    #
    #     @turbo for i=1:n
    #         χ2 += (x[i] - wx) * (x[i] - wx) / (σ[i] * σ[i])
    #     end
    #     mswd = χ2 / (n-1)
    #     wσ = sqrt(mswd / sum_of_weights)
    #     return wx, wσ, mswd
    # end
    export gwmean

    """
    ```julia
    MSWD(x, σ)
    ```
    Return the Mean Square of Weighted Deviates (AKA the reduced chi-squared
    statistic) of a dataset with values `x` and one-sigma uncertainties `σ`
    """
    function MSWD(x, σ)
        sum_of_values = sum_of_weights = χ2 = 0.0
        n = length(x)

        @inbounds @simd for i=1:n
            w = 1 / (σ[i]*σ[i])
            sum_of_values += w * x[i]
            sum_of_weights += w
        end
        wx = sum_of_values / sum_of_weights

        @inbounds @simd for i=1:n
            χ2 += (x[i] - wx) * (x[i] - wx) / (σ[i] * σ[i])
        end

        return χ2 / (n-1)
    end
    # function MSWD(x::Array{<:Number}, σ::Array{<:Number})
    #     sum_of_values = sum_of_weights = χ2 = 0.0
    #     n = length(x)
    #
    #     @turbo for i=1:n
    #         w = 1 / (σ[i]*σ[i])
    #         sum_of_values += w * x[i]
    #         sum_of_weights += w
    #     end
    #     wx = sum_of_values / sum_of_weights
    #
    #     @turbo for i=1:n
    #         χ2 += (x[i] - wx) * (x[i] - wx) / (σ[i] * σ[i])
    #     end
    #
    #     return χ2 / (n-1)
    # end
    export MSWD

## -- The York (1968) two-dimensional linear regression with x and y uncertainties
    # as commonly used in isochrons

    # Custom type to hold York fit resutls
    struct YorkFit{T}
        intercept::T
        intercept_sigma::T
        slope::T
        slope_sigma::T
        mswd::T
    end

    """
    ```julia
    yorkfit(x, σx, y, σy)
    ```
    Uses the York (1968) two-dimensional least-squares fit to calculate `a`, `b`,
    and uncertanties `σa`, `σb` for the equation `y = a + bx`, given `x`, `y` and
    uncertaintes `σx`, ``σy`.
    """
    function yorkfit(x, σx, y, σy; niterations=10)

        ## 1. Ordinary linear regression (to get a first estimate of slope and intercept)

        # Check for missing data
        t = (x.==x) .& (y.==y) .& (σx.==σx) .& (σy.==σy)
        x = x[t]
        y = y[t]
        σx = σx[t]
        σy = σy[t]

        # Calculate the ordinary least-squares fit
        # For the equation y=a+bx, m(1)=a, m(2)=b
        g = [ones(length(x)) x]
        m = (g'*g)\g'*y
        b = m[2]
        a = m[1]

        ## 2. Now, let's define parameters needed by the York fit

        # Weighting factors
        ωx = 1.0 ./ σx.^2
        ωy = 1.0 ./ σy.^2

        # terms that don't depend on a or b
        α = sqrt.(ωx .* ωy)

        x̄ = _vmean(x, :)
        ȳ = _vmean(y, :)
        r = sum((x .- x̄).*(y .- ȳ)) ./ (sqrt(sum((x .- x̄).^2)) * sqrt(sum((y .- ȳ).^2)))

        ## 3. Perform the York fit (must iterate)
        W = ωx.*ωy ./ (b^2*ωy + ωx - 2*b*r.*α)

        X̄ = sum(W.*x) / sum(W)
        Ȳ = sum(W.*y) / sum(W)

        U = x .- X̄
        V = y .- Ȳ

        sV = W.^2 .* V .* (U./ωy + b.*V./ωx - r.*V./α)
        sU = W.^2 .* U .* (U./ωy + b.*V./ωx - b.*r.*U./α)
        b = sum(sV) ./ sum(sU)

        a = Ȳ - b .* X̄
        for i = 2:niterations
            W .= ωx.*ωy ./ (b^2*ωy + ωx - 2*b*r.*α)

            X̄ = sum(W.*x) / sum(W)
            Ȳ = sum(W.*y) / sum(W)

            U .= x .- X̄
            V .= y .- Ȳ

            sV .= W.^2 .* V .* (U./ωy + b.*V./ωx - r.*V./α)
            sU .= W.^2 .* U .* (U./ωy + b.*V./ωx - b.*r.*U./α)
            b = sum(sV) ./ sum(sU)

            a = Ȳ - b .* X̄
        end

        ## 4. Calculate uncertainties and MSWD
        β = W .* (U./ωy + b.*V./ωx - (b.*U+V).*r./α)

        u = X̄ .+ β
        v = Ȳ .+ b.*β

        xm = sum(W.*u)./sum(W)
        ym = sum(W.*v)./sum(W)

        σb = sqrt(1.0 ./ sum(W .* (u .- xm).^2))
        σa = sqrt(1.0 ./ sum(W) + xm.^2 .* σb.^2)

        # MSWD (reduced chi-squared) of the fit
        mswd = 1.0 ./ length(x) .* sum( (y .- a.-b.* x).^2 ./ (σy.^2 + b.^2 .* σx.^2) )

        ## Results
        return YorkFit(a, σa, b, σb, mswd)
    end
    export yorkfit


# Gini.jl

###### gini #####
"""
    gini(v)

Compute the Gini Coefficient of a vector `v` .

# Examples
```julia
julia> using Inequality
julia> gini([8, 5, 1, 3, 5, 6, 7, 6, 3])
0.2373737373737374
```
"""
gini(v::AbstractVector{<:Real})::Float64 = (2 * sum([x*i for (i,x) in enumerate(sort(v))]) / sum(sort(v)) - (length(v)+1))/(length(v))

###### weighted gini #####
"""
    gini(v, w)

Compute the weighted Gini Coefficient of a vector `v` using weights given by a weight vector `w`.

Weights must not be negative, missing or NaN. The weights and data vectors must have the same length.

# Examples
```julia
julia> using Inequality
julia> gini([8, 5, 1, 3, 5, 6, 7, 6, 3], collect(0.1:0.1:0.9))
0.20652395514780775
```
"""
function gini(v::AbstractVector{<:Real}, w::AbstractVector{<:Real})::Float64

    checks_weights(v, w)

    w = w[sortperm(v)]/sum(w)
    v = sort(v)
    p = cumsum(w)
    n = length(v)
    nᵤ = cumsum(w .* v)/cumsum(w .* v)[end]
    sum(nᵤ[2:end] .* p[1:(end-1)]) - sum(nᵤ[1:(end-1)] .* p[2:end])

end

function gini(v::AbstractVector{<:Real}, w::AbstractWeights)::Float64

    checks_weights(v, w)

    w = w[sortperm(v)]/w.sum
    v = sort(v)
    p = cumsum(w)
    n = length(v)
    nᵤ = cumsum(w .* v)/cumsum(w .* v)[end]
    sum(nᵤ[2:end] .* p[1:(end-1)]) - sum(nᵤ[1:(end-1)] .* p[2:end])

end

"""
    wgini(v, w)

Compute the Gini Coefficient of `v` with weights `w`. See also [`gini`](@gini)
"""
wgini(v::AbstractVector{<:Real}, w::AbstractVector{<:Real}) = gini(v, w)

wgini(v::AbstractVector{<:Real}, w::AbstractWeights) = gini(v, w)


function _make_radii(a::Number, b::Number, c::Number)
    dab = abs(a - b)
    dbc = abs(b - c)
    dac = abs(a - c)
    rhs = [dab; dbc; dac]
    A = [1 1 0; 0 1 1; 1 0 1]

    radii = A \ rhs
end

"""
    kiss(z1,z2,z3)

Given three points (as complex numbers) find three circles
centered at those points that are mutually tangent.

See: `touch_points`
"""
function kiss(a::Number, b::Number, c::Number)
    rads = _make_radii(a, b, c)
    ctrs = [a, b, c]

    [Circle(ctrs[k], rads[k]) for k = 1:3]
end

export kiss, touch_points

"""
    touch_points(z1,z2,z3)

Given three points (as complex numbers) determine the points of tangency
of three mutually tangent circles centered at those points.

See: `kiss`
"""
function touch_points(a::Number, b::Number, c::Number)
    radii = _make_radii(a, b, c)
    ra, rb, rc = radii

    ab = (b - a) * (ra / (ra + rb)) + a
    ac = (c - a) * (ra / (ra + rc)) + a
    bc = (c - b) * (rb / (rb + rc)) + b
    return [ab; ac; bc]
end

"""
    soddy_radii(r1, r2, r3)
Given three radii (real numbers) of mutually tangent circles,
give the radii of a fourth circle that is mutually tangent to the
three that are given as a 2-tuple of real numbers.

The positive result (first entry) is the radius of a small circle
nestled between the other three, and the negative result (second entry)
is negative the radius of a circle that is tangent to and surrounds the
three.
"""
function soddy_radii(r1, r2, r3)
    k1 = 1 / r1
    k2 = 1 / r2
    k3 = 1 / r3

    common = 2 * sqrt(k1 * k2 + k2 * k3 + k3 * k1)

    k4a = k1 + k2 + k3 + common
    k4b = k1 + k2 + k3 - common

    return 1 / k4a, 1 / k4b
end

soddy_radii(C1::Circle, C2::Circle, C3::Circle) = soddy_radii(C1.r, C2.r, C3.r)

function soddy_center(C1::Circle, C2::Circle, C3::Circle)
    r = soddy_radii(C1, C2, C3)[1]

    D1 = Circle(center(C1), radius(C1) + r)
    D2 = Circle(center(C2), radius(C2) + r)

    zz = collect(D1 ∩ D2)

    d1 = abs(center(C3) - zz[1]) - (radius(C3) + r) |> abs
    d2 = abs(center(C3) - zz[2]) - (radius(C3) + r) |> abs

    d1 < d2 ? zz[1] : zz[2]

end

export soddy

"""
    soddy(C1,C2,C3)
Given three mutually tangent circles, return the circle that is
tangent to all three an nesteled between them.
"""
function soddy(C1::Circle, C2::Circle, C3::Circle)
    r = soddy_radii(C1, C2, C3)[1]
    z = soddy_center(C1, C2, C3)
    return Circle(z, r)
end


function EuropeanCallMC_Antithetic(K, T, S, sig, r, div, N, M)

## setting parameters

dt = T/N
nudt = (r - div - 0.5 * sig^2) * dt
sigsdt = sig * sqrt(dt)
lnS = log(S)

## variables to store the sums of the option values

sum_CT = 0
sum_CT2 = 0

## outer loop is the number of simulations bigger is better

for j = 1:M

    ## setting up two path variables
  lnSt1 = lnS
  lnSt2 = lnS

  for i = 1:N

      # antithetic variance reduction through the perectly negative correlated error term
    error = randn()
    lnSt1 = lnSt1 + nudt + sigsdt * (error)
    lnSt2 = lnSt2 + nudt + sigsdt * (-error)

  end

    ## wraps up and stores all the values of each path

  ST1 = exp(lnSt1)
  ST2 = exp(lnSt2)
  CT = 0.5 * ( max(0, ST1 - K) + max(0, ST2 - K))
  sum_CT = sum_CT + CT
  sum_CT2 = sum_CT2 + CT^2

end

  ## discounting the average call value
call_value = sum_CT / M * exp(-r*T)

SD = (sqrt(sum_CT2 - 1/M * (sum_CT^2)) * exp(-2*r*T)) / (M-1)
SE = SD / sqrt(M)

println("The Call Value is $call_value")
println("The SD is $SD")
println("The SE is $SE")

MC = [call_value, SD, SE]

end


using Random, Distributions, HypothesisTests, Plots; pyplot()

muRange = 51:0.02:55
n = 20
N = 10^4
mu0 = 53.0
powerT, powerU = [], []

for muActual in muRange

    dist = Normal(muActual, 1.2)
    rejectT, rejectU = 0, 0
    Random.seed!(1)

    for _ in 1:N
        data = rand(dist,n)
        xBar, stdDev = mean(data), std(data)

        tStatT = (xBar - mu0)/(stdDev/sqrt(n))
        pValT  = 2*ccdf(TDist(n-1), abs(tStatT))

        xPositive = sum(data .> mu0)
        uStat = max(xPositive, n-xPositive)
        pValSign = 2*cdf(Binomial(n,0.5), uStat)

        rejectT += pValT < 0.05
        rejectU += pValSign < 0.05
    end

    push!(powerT, rejectT/N)
    push!(powerU, rejectU/N)

end

plot(muRange, powerT, c=:blue, label="t test")
plot!(muRange, powerU, c=:red, label="Sign test",
	xlims=(51,55), ylims=(0,1),
	xlabel="Different values of muActual",
	ylabel="Proportion of times H0 rejected", legend=:bottomleft)


using Roots, Plots

f(x) = x^3 - x - 1
@show fzeros(f, -10, 10)
println()

y = -10:0.1:10
plot(y, f.(y))

@show fzero(f, [-2,2])
println()

@show fzero(f, 1.0)
println()

f(x) = exp(x) - x^4
fp(x) = exp(x) - 4x^3
fpp(x) = exp(x) - 12x^2

Roots.newton(f, fp, 8)        # 8.613169456441398

halley(f, fp, fpp, 8)

secant_method(f, [8, 8.5])


struct Pulse{T,F}
    Tp :: T
    f  :: F
end

(p::Pulse)(t) = p.f(t)

"""cosinepulse(T ; normalize = true)

A cosine pulse of duration `T`. This pulse is also known as a "time-domain
raised cosine" pulse."""

function cosinepulse(T ; normalize = true)
    p1(t) = begin
        if 0 ≤ t < T
            return 1.0-cospi(2*t/T)
        else
            return 0.0
        end
    end
    N = 1.0
    if normalize
        area, _ = quadgk(t -> p1(t)^2, 0.0, T)
        N = 1.0/sqrt(area)
    end
    return Pulse(T, t -> N*p1(t))
end

"""halfinepulse(T ; normalize = true)

A half-sine pulse of duration `T`."""
function  halfsinepulse(T ; normalize = true)
    p1(t) = begin
        if 0 ≤ t < T
            return sinpi(t/T)
        else
            return 0.0
        end
    end
    N = 1.0
    if normalize
        area, _ = quadgk(t -> p1(t)^2, 0.0, T)
        N = 1.0/sqrt(area)
    end
    return Pulse(T, t -> N*p1(t))
end

"""
    rcpulse(T, β, duration = 10T ; normalize = true)

A causal raised-cosine pulse with main lobe duration `T`, roll-off
factor (excess bandwidth) `β`, and duration `d`.
"""
function rcpulse(T, β, duration = 10T ; normalize = true)
    if β > 1 || β < 0
        @error "Excess bandwidth β must be between 0 and 1, inclusive." β
    end
    p1(t) = begin
        t ≥ duration/2 && return 0.0
        t < -duration/2 && return 0.0
        β == 0 && return sinc(t)
        if t ≈ T/(2β) || t ≈ -T/(2β)
            return (π/(4T))*sinc(1/(2β))
        end
        return (1/T)*sinc(t/T)*cospi(β*t/T)/(1-(2β*t/T)^2)
    end
    N = 1.0
    if normalize
        area, _ = quadgk(t -> p1(t)^2, -duration/2, duration/2)
        N = 1.0 / sqrt(area)
    end
    return Pulse(T, t -> N * p1(t - duration/2))
end

"""
    srrcpulse(T, β, duration = 10T ; normalize = true)

A causal square-root raised cosine pulse with main lobe duration `T`, roll-off
factor (excess bandwidth) `β``, and duration `d`.
"""
function srrcpulse(T, β, duration = 10T ; normalize = true)
    if β > 1 || β < 0
        @error "Excess bandwidth β must be between 0 and 1, inclusive." β
    end
    p1(t) = begin
        t ≥ duration/2 && return 0.0
        t < -duration/2 && return 0.0
        β == 0 && return sinc(t)
        t ≈ 0 && return (1 - β + 4β / π) / sqrt(T)
        if t ≈ T / (4β) || t ≈ -T / (4β)
            x = (β * (π + 2)) / (π * sqrt(2T)) * sinpi(1 / (4β))
            y = (β * (π - 2)) / (π * sqrt(2T)) * cospi(1 / (4β))
            return x + y
        else
            x = sinpi((1 - β) * t / T)
            y = (4β * t / T) * cospi((1 + β) * t / T)
            z = sqrt(T) * (π * t / T) * (1 - (4β * t / T)^2)
            return (x + y) / z
        end
    end
    N = 1.0
    if normalize
        area, _ = quadgk(t -> p1(t)^2, -duration/2, duration/2)
        N = 1.0 / sqrt(area)
    end
    return Pulse(T, t -> N * p1(t - duration/2))
end

"""gaussianpulse(B, T, duration=10 ; normalize = true)

A causal gaussian pulse with 3-dB bandwidth `B`, symbol time
`T`, and the specified `duration`.

See Molisch, "Wireless Communications", 11.2.2 and 11.3.9.
"""
function gaussianpulse(B, T, duration = 4T; normalize = true)
    p1(t) = begin
        t ≥ duration/2 && return 0.0
        t < -duration/2 && return 0.0
        σ = sqrt(log(2))/(2π*B*T)
        return 1/(sqrt(2π)*σ*T)*exp(-t^2/(2σ^2*T^2))
    end
    N = 1.0
    if normalize
        area, _ = quadgk(t -> p1(t)^2, -duration / 2, duration / 2)
        N = 1.0 / sqrt(area)
    end
    return Pulse(T, t -> N * p1(t - duration / 2))
end


export Unitary, is_unitary

"""
    is_unitary( U :: Matrix; atol=ATOL :: Float64 ) :: Bool

Returns `true` if matrix `U` is unitary. The hermitian adjoint of a unitary matrix
is its inverse:
* `U' * U == I` where `I` is the identity matrix.
"""
function is_unitary(U::AbstractMatrix; atol=ATOL :: Float64) :: Bool
    if !isequal(size(U)...)
        return false
    end
    dim = size(U,1)

    isapprox(U'*U, Matrix(I,dim,dim), atol=atol)
end

"""
    Unitary( U :: AbstractMatrix ) <: Operator{T}

Unitary matrices represent the physical evolution of quantum states. The Constructor,
`Unitary(U)`, throws a `DomainError` if the provided matrix, `U` is not unitary.
"""
struct Unitary{T<:Number} <: Operator{T}
    M :: Matrix{T}
    atol :: Float64
    Unitary(
        U :: AbstractMatrix{<:Number};
        atol=ATOL :: Float64
    ) = is_unitary(U, atol=atol) ? new{eltype(U)}(U, atol) : throw(DomainError(U, "matrix U is not unitary"))
end
is_unitary(U :: Unitary) = true

"""
    *(unitaries :: Vararg{Unitary}; atol=ATOL :: Float64) :: Unitary
"""
*(unitaries :: Vararg{Unitary}; atol=ATOL :: Float64) :: Unitary = Unitary(*(map(U -> U.M, unitaries)...), atol=atol)

"""
    kron(unitaries :: Vararg{Unitary}; atol=ATOL :: Float64) :: Unitary
"""
kron(unitaries :: Vararg{Unitary}; atol=ATOL :: Float64) :: Unitary = Unitary(kron(map(U -> U.M, unitaries)...), atol=atol)

"""
    adjoint(U :: Unitary) :: Unitary
"""
adjoint(U :: Unitary) :: Unitary = Unitary(U.M', atol=U.atol)


# Set random seed for the random generator
#  so that our results are reproducible

function gibs(nrep=105000;
              seed=123,
              nb  =5000,
              ρ   =.5)
   # Seed - sets the random number generator seed
   #   so that it is easy to create reproducible results.
   # nrep - in the number of repetitions of the MCMC draws.
   # nb   - number of observations for the "Burn-in"
   # ρ    - Correlation between Y1 and Y2
   srand(seed)

   # Create empty vectors of length nrep
   yy₁ = yy₂ = spzeros(nrep,1)

   # Calculate standard deviation of Y2
   σ   = sqrt(1-ρ^2)

   # Initialize y1 with a single normal draw
   y₁ = randn()*σ

   # Now we start the Gibbs sampler loop
   for i in 1:nrep
     y₂ = randn()*σ + ρ*y₁
     y₁ = randn()*σ + ρ*y₂
     yy₁[i]=y₁
     yy₂[i]=y₂
   end

   return (yy₁[(nb+1):end,1],yy₂[(nb+1):end,1])
end
gibs()

# Set random seed for the random generator
#  so that our results are reproducible
function gibs2(nrep=105000;
               seed=123,
               nb  =5000,
               ρ   =.5)
   # Seed - sets the random number generator seed
   #   so that it is easy to create reproducible results.
   # nrep - in the number of repetitions of the MCMC draws.
   # nb   - number of observations for the "Burn-in"
   # ρ    - Correlation between Y1 and Y2
   srand(seed)

   # Calculate standard deviation of Y2
   sigma = sqrt(1-ρ^2)

   # Create empty vectors of length nrep
   y1 = randn(nrep)
   y2 = randn(nrep)

   # Now we start the Gibbs sampler loop
   for i in 2:nrep
     y2[i] += y1[i-1]*ρ
     y1[i] += y2[i]*ρ
   end

   return (y1[(nb+1):end,1],y2[(nb+1):end,1])
end
gibs2()

@elapsed yy = gibs()
# 0.063151467
@elapsed gibs(10^6)
# 0.479542057

@elapsed yy = gibs2()
# 0.010729382
@elapsed yy = gibs2(10^6)
# 0.065821774


"""`bisquareWM(x, k, center, tol)`

Return the bisquare weighted mean, a location estimator.
The bisquare weighted mean of the data `x` with a k-value of `k`.
A sensible choice of `k` is 3 to 5 times the rms width or 1.3 to 2 times the
full width at half max of a peak.  For strictly Gaussian data, the choices of
k= 3.14, 3.88, and 4.68 times sigma will be 80%, 90%, and 95% efficient.
Data values a distance of more than `k` from the weighted mean are given no
weight.

`center` is used as an initial guess at the weighted mean; if omitted,
then the data median will be used.

The answer is found iteratively, revised until it changes by less than `tol`.  If
`tol` is omitted, then `tol` will use 1e-5 times the median absolute
deviation of `x` about its median."""
function bisquareWM(x::AbstractArray{T}, k::Real, center::Real, tol::Real) where {T <: Real}
    for _iteration in 1:100
        weights = (1.0-((x-center)/k).^2).^2
        weights[abs.(x-center).>k] = 0.0
        newcenter = dot(weights, x)/sum(weights)
        if abs(newcenter - center)<tol
            return newcenter
        end
        center = newcenter
    end
    error("bisquareWM used too many iterations.\nConsider using higher `tol` or better `center`, or change to trimean(x).")
end

bisquareWM(x::AbstractArray{T}, k::Real, center::Real) where {T<:Real} =
        bisquareWM(x, k, center, 1e-5*StatsBase.mad(float(x)))
bisquareWM(x::AbstractArray{T}, k::Real) where {T<:Real} = bisquareWM(x, k, median(x))

"""huberWM(x, k, center, tol)`

Return Huber's weighted mean, a location estimator.
Huber's weighted mean of the data `x` with a k-value of `k`.
A sensible choice of `k` is 1 to 1.5 times the rms width or 0.4 to 0.6 times the
full width at half max of a peak.  For strictly Gaussian data, the choices of
k=1.0 and 1.4 sigma give ...
Data values a distance of more than `k` from the weighted mean are given no
weight.

`center` is used as an initial guess at the weighted mean.
If `center` is None, then the data median will be used.

The answer is found iteratively, revised until it changes by less than `tol`.  If
`tol` is None (the default), then `tol` will use 1e-5 times the median absolute
deviation of `x` about its median."""
function huberWM(x::AbstractArray{T}, k::Real, center::Real, tol::Real) where {T <: Real}
    for _iteration = 1:100
        weights = float(k)./abs.(x-center)
        weights[weights .> 1.0] = 1.0
        newcenter = dot(weights, x)/sum(weights)
        if abs(newcenter - center) < tol
            return newcenter
        end
        center = newcenter
    end
    error("huberWM used too many iterations.\nConsider using higher `tol` or better `center`, or change to trimean(x).")
end

huberWM(x::AbstractArray{T}, k::Real, center::Real) where {T<:Real} =
        huberWM(x, k, center, 1e-5*StatsBase.mad(float(x)))
huberWM(x::AbstractArray{T}, k::Real) where {T<:Real} = huberWM(x, k, median(x))

"""`trimean(x)`

Return Tukey's trimean, a location estimator, for a data set `x`.

If (q1,q2,q3) are the quartiles (i.e., the 25%ile, median, and 75 %ile),
the trimean is (q1+q3)/4 + q2/2. Uses `idealf(x)` for finding quartiles.
"""
function trimean(x::AbstractArray{T}) where {T <: Real}
    mean([mean(idealf(x)), median(x)])
end


function binomial(n::Int,m::Int)
	# \Choose{n}{m}
	ans=1
	for i=1:m
		ans *=(n+1-i)
		ans/=i
		ans=Int(ans)
	end
	return ans

end
"""
# test
for i=0:10
println(binomial(10,i))
end
"""


# auxiliary simulation functions
#
@doc """
Simulates a stochastic process {yₜ} following an ARMA(1,1) with parameters φ and Θ for t = 1, ..., T periods.

```math
  y_t = \\mu_0 + \\mu_1 + \\phi*y_{t-1} + \\epsilon_t + \\theta*\\epsilon_{t-1}, \\quad \\epsilon_t \\sim \\mathbf{N}(0, \\sigma^2)
```

Required Input Parameters
----------------------------------
none

Optional Input Parameters
----------------------------------
`mu::Float64`: AR(1) process constant term. (default=0.0)

`lin_trend::Float64`: AR(1) process linear time trend. (default=0.0)

`ar::Float64`: AR(1) autorregressive parameter. (default=0.5)

`ma::Float64`: MA(1) autorregressive parameter. (default=0.5)

`T::Int64`: Length of simulated time series. (default=100)

`y_0::Float64`: Initial value of the series. (default=0.0)

`stderr::Float64` : Normally distributed innovation standard error. (default=1.0)

Output
----------------------------------
`y::Vector{Float64}`: Simulated ARMA(1,1) time series of length T.

""" ->
function simul_arma(;mu::Float64 = 0., lin_trend::Float64 = 0., ar::Float64 = .5, ma::Float64 = .5, T::Integer = 100, y_0::Float64 = 0.0, stderr::Float64 = 1.0)
  #y_t = μ + ϕy_t-1 + u_t/ u_t = ɛ_t + Θɛ_t-1
  y = zeros(T); ɛ = rand(Normal(0.0,stderr),T); u = zeros(T);
  for t in 2:T
    u[t, :] = ɛ[t, :] + ma*ɛ[t-1, :] #MA
    y[t, :] = mu + lin_trend*t + ar*y[t-1, :] + u[t, :] #AR
  end
  return y
end
#
#
function simul_llm(; T=100::Int64, x0=0.0::Float64)
  # x_t = μ_t + ϵ_t, ϵ_t∼N(0, σ2_ϵ)
  # μ_t = μ_{t-1} + η_t, ϵ_t∼N(0, σ2_η)
  #state and signal initalized at zero
  x = zeros(T)
  μ = zeros(T)
  σ2_ϵ = 1.0;
  σ2_η = 1.0;
  ϵ = rand(Normal(0.0, σ2_ϵ), T)
  η = rand(Normal(0.0, σ2_η), T)
  x[1] = x0
  for t in 2:T
    μ[t] = μ[t-1] + η[t]
    x[t] = μ[t] + ϵ[t]
  end
  return x
end


################################################################################
# Breast Cancer diagnosis problem
#
# In this problem we are given a dataset containing real world characteristics of observed breast cancer (size, pattern,..) together with the associated diagnosis in terms of malignity or benignity of the cancer.
# Our task is to build a linear classifier using the perceptron algorithm that we studied and train it in order to make diagnosis based on the cancer characteristics.

# ### Environment set-up and data loading

# 1) Start by setting the working directory to the directory of this file and activate it. If you have the provided `Manifest.toml` file in the directory, just run `Pkg.instantiate()`, otherwise manually add the packages Pipe, HTTP, StatsPlots and BetaML.
# Also, seed the random number generator with the integer `123`.

# 2) Load the packages/modules Statistics, DelimitedFiles, LinearAlgebra, Pipe, HTTP, StatsPlots, BetaML

# 3) Load from internet or from localfile the input data and shuffle its rows (records)

dataURL = "https://raw.githubusercontent.com/sylvaticus/IntroSPMLJuliaCourse/main/lessonsMaterial/03_ML1/BreastCancerDiagnosisWithPerceptron/data/wdbc.data.csv"

# Source: [Breast Cancer Wisconsin (Diagnostic) Data Set, UCI Machine Learning Repository](https://archive.ics.uci.edu/ml/datasets/Breast+Cancer+Wisconsin+%28Diagnostic%29)

# 4) Map the data to (X,y)
# The data you have loaded contains the actual diagnosis for the cancer in the second column, coded with a string "B" for "Benign" and "M" for "Malign", and the characteristics of the cancer foir the next 30 columns.
# Save the diagnosis to the vector `y`, coding malign cancers with `1` and benign cancers with `-1`
# Save the characteristics to the feature matrix `X` (and be sure it is made of Float64)

# 5) (this task is provided) Plot the first 2 attributes of training points and define the function plot2DClassifierWithData()
colors = [y == 1 ? "red" : "green" for y in y]
labels = [y == 1 ? "malign" : "benign" for y in y]
scatter(X[:,1],X[:,2], colour=colors, title="Classified tumors",xlabel="Tumor Radius", ylabel="Tumor Texture", group=labels)

function plot2DClassifierWithData(X,y,θ;d1=1,d2=2,origin=false,xlabel="Dimx: $(d1)",ylabel="Dimy: $(d2)")
    nR    = size(X,1)
    X     = hcat(ones(nR),X)
    X     = scale(X) # for visualisation
    d1    += 1
    d2    += 1
    colors = [y == 1 ? "red" : "green" for y in y]
    labels = [y == 1 ? "malign" : "benign" for y in y]
    minD1,maxD1 = extrema(X[:,d1])
    minD2,maxD2 = extrema(X[:,d2])
    myplot = scatter(X[:,d1],X[:,d2], colour=colors, title="Linear classifier in 2D",xlabel=xlabel, ylabel=ylabel, group=labels)
    d2Class(x) = -θ[1]/θ[d2] -x * θ[d1]/θ[d2]
    if θ[d2] == 0
        vline!([0], color= "blue",label="",linewidth=5)
    else
        plot!(d2Class,minD1,maxD1, color= "blue",label="",linewidth=5)
    end
    display(myplot)
end

# 6) (provided) Define the Model and training options structures

abstract type SupervisedModel end
abstract type TrainingOptions end

mutable struct Perceptron <: SupervisedModel
    θ::Vector{Float64}
end

mutable struct PerceptronTrainingOptions <: TrainingOptions
    epochs::Int64
    verbose::Bool
    shuffle::Bool
    function PerceptronTrainingOptions(;epochs=1,verbose=false,shuffle=false)
        return new(epochs,verbose,shuffle)
    end
end

# 7) (provided) Implement the functions `predict()` and `update()`
function predict(model::Perceptron,x::AbstractVector)
    x = vcat(1.0,x)
    x' * model.θ > eps() ? (return 1) : (return -1)
end

function predict(model::Perceptron,X::AbstractMatrix)
    return [predict(model,r) for r in eachrow(X)]
end

function update!(model::Perceptron,X::Vector,y)
    X       = vcat(1.0,X)
    model.θ = model.θ .+ y .* X
    return model.θ
end

# 8) Implement the function `train!(model::Perceptron,X,y,ops=PerceptronTrainingOptions()::TrainingOptions)`
# Compared to the function we saw in the 0302-perceptron.jl file, add, if you wish, a counter to eventually return early if there are no more errors in an epoch (i.e., all points are correctly classified)
function train!(model::Perceptron,X,y,ops=PerceptronTrainingOptions()::TrainingOptions)
    #...
    for t in 1:epochs
        #...
        if ops.shuffle
           #...
        end
        for n in 1:nR
           #...
        end
        #...
    end
    #...
    return model.θ
end

# 9) Instanziate a `Perceptron` object with a parameter vector of nD+1 zeros and a `PerceptronTrainingOption` object with 5 epochs and shuffling, use the options to train the model on the whole dataset, compute the model predictions and the accuracy relative to the whole sample.

m   = Perceptron(zeros(size(X,2)+1))
ops = #...
train!(m,X,y,ops)
plot2DClassifierWithData(X,y,m.θ,d1=1,d2=2,xlabel="Tumor Radius", ylabel="Tumor Texture")
ŷ           = #...
inSampleAcc = accuracy(#= ... =#) # 0.91

# 10) Partition the data in `(xtrain,xtest)` and `(ytrain,ytest)` keeping 65% of the data for training and reserving 35% for testing

((xtrain,xtest),(ytrain,ytest)) = partition(#=...=#)

# 11) Using a 10-folds cross-validation strategy, find the best hyperparameters within the following ranges :

sampler = KFold(#=...=#)

epochsSet  = 1:5:150
shuffleSet = [false,true]

bestE       = 0
bestShuffle = false
bestAcc     = 0.0

accuraciesNonShuffle = []
accuraciesShuffle    = []

for e in epochsSet, s in shuffleSet
    global bestE, bestShuffle, bestAcc, accuraciesNonShuffle, accuraciesShuffle
    local acc
    local ops  = PerceptronTrainingOptions(#=...=#)
    (acc,_)    = crossValidation([xtrain,ytrain],sampler) do trainData,valData,rng
                    (xtrain,ytrain) = trainData; (xval,yval) = valData
                    m               = Perceptron(zeros(size(xtrain,2)+1))
                    train!(#=...=#)
                    ŷval            = predict(#=...=#)
                    valAccuracy     = accuracy(#=...=#)
                    return valAccuracy
                end
    if acc > bestAcc
        bestAcc     = acc
        bestE       = e
        bestShuffle = s
    end
    if s
        push!(accuraciesShuffle,acc)
    else
        push!(accuraciesNonShuffle,acc)
    end
end

bestAcc # 0.91
bestE
bestShuffle

plot(epochsSet,accuraciesNonShuffle,label="Val accuracy without shuffling", legend=:bottomright)
plot!(epochsSet,accuraciesShuffle, label="Val accuracy with shuffling")

# 12) Using the "best" hyperparameters found in the previous step, instantiate a new model and options, train the model using `(xtrain,ytrain)`, make your predicitons for the testing features (`xtest`) and compute your output accuracy compared with those of the true `ytest` (use the BetaML function `accuracy`)

ops = PerceptronTrainingOptions(#=...=#)
m   = Perceptron(zeros(size(xtest,2)+1))
train!(#=...=#)
ŷtest           = predict(#=...=#)
testAccuracy    = accuracy(#=...=#) # 0.89

plot2DClassifierWithData(xtest,ytest,m.θ,xlabel="Tumor Radius", ylabel="Tumor Texture")
plot2DClassifierWithData(xtest,ytest,m.θ,d1=3,d2=4)

# 13) (optional) Add a scaling passage to the workflow and test it with cross-validation if it improves the accuracy

epochsSet  = 1:10:150
shuffleSet = [false,true]
scalingSet = [false,true]

bestE       = 0
bestShuffle = false
bestScaling = false
bestAcc     = 0.0

accNShuffleNSc = Float64[]
accNShuffleSc  = Float64[]
accShuffleNSc  = Float64[]
accShuffleSc   = Float64[]

scfactors = getScaleFactors(xtrain)

for e in epochsSet, s in shuffleSet, sc in scalingSet
    global bestE, bestShuffle, bestAcc, accNShuffleNSc, accNShuffleSc, accShuffleNSc, accShuffleSc
    local acc
    local ops  = PerceptronTrainingOptions(#=...=#)
    xtrainsc= copy(xtrain)
    if(sc)
        xtrainsc =scale(xtrain,scfactors)
    end
    (acc,_)    = crossValidation([xtrainsc,ytrain],sampler) do trainData,valData,rng
                    #...
                    return valAccuracy
                end
    if acc > bestAcc
        bestAcc     = acc
        bestE       = e
        bestShuffle = s
        bestScaling = sc
    end
    if s && sc
        push!(accShuffleSc,acc)
    elseif  s && !sc
        push!(accShuffleNSc,acc)
    elseif  !s && sc
        push!(accNShuffleSc,acc)
    elseif  !s && !sc
        push!(accNShuffleNSc,acc)
    else
        @error "Something wrong here"
    end
end

bestAcc # 0.97
bestE
bestShuffle
bestScaling

plot(epochsSet,accShuffleSc,label="Val accuracy shuffling scaling", legend=:bottomright)
plot!(epochsSet,accNShuffleSc,label="Val accuracy NON shuffling scaling", legend=:bottomright)
plot!(epochsSet,accShuffleNSc,label="Val accuracy shuffling NON scaling", legend=:bottomright)
plot!(epochsSet,accNShuffleNSc,label="Val accuracy NON shuffling NON scaling", legend=:bottomright)

ops = PerceptronTrainingOptions(#=...=#)
m   = Perceptron(#=...=#)
if bestScaling
    train!(m,scale(xtrain),ytrain,ops)
    ŷtest  = predict(m,scale(xtest))
else
    train!(m,xtrain,ytrain,ops)
    ŷtest  = predict(m,xtest)
end
testAccuracy    = accuracy(#=...=#) # 0.96

plot2DClassifierWithData(xtest,ytest,m.θ,xlabel="Tumor Radius", ylabel="Tumor Texture")


abstract type Tax end

net(x, T::Tax)=x-tax(x, T)
gross(x, T::Tax)=x+tax(x, T)

# * Gouveia and Strauss (1994)
struct GouveiaStrauss <: Tax
    maxmargtax :: Float64
    coefexp    :: Float64
    coefcons   :: Float64
end

function GouveiaStrauss(;maxmargtax=0.275,
                        coefexp=5.,
                        coefcons=0.3)
    GouveiaStrauss(maxmargtax, coefexp, coefcons)
end

tax(y, gs::GouveiaStrauss)=_gouveiastrauss(y, gs.maxmargtax, gs.coefexp, gs.coefcons)

@doc doc"
    _gouveiastrauss(y, φ₀, φ₁, φ₂)

Inspired by Gouveia and Strauss (1994)

Parametrizes income taxes with a differentiable function characterized by three
parameters (the phis). If
```math
  \tau_y(y) = \phi_0 \left[ y - \left( y^{-\phi_1} + \phi_2  \right)^{-\frac{1}{\phi_1}} \right]
```

The $\phi_0$ coefficient is the limit $\lim \tau_y^{\prime}(y)$ when
$y\to\infty$. The other coefficients govern the degree of progressiveness of the
function. (For example, when $\phi_2 \to 0$, no one is taxed.)
"
_gouveiastrauss(y, φ₀, φ₁, φ₂)=φ₀*(y-(y^(-φ₁)+φ₂)^(-1/φ₁))

# * Simple linear schedule

struct AdValorem <: Tax
    t::Float64
end

tax(y, lav::AdValorem)=y*lav.t

# * Null tax

struct NullTax <: Tax end
tax(y, ::NullTax)=zero(y)


"Square the sum of the first `n` positive integers"
function square_of_sum(n)
    (n*(n +1 ) ÷ 2)^2
end

"Sum the squares of the first `n` positive integers"
function sum_of_squares(n)
    (n*(n + 1)*(2n + 1)) ÷ 6

end

"Subtract the sum of squares from square of the sum of the first `n` positive ints"
function difference(n)
    square_of_sum(n)-sum_of_squares(n)
end


using SymPy

x, y, t = @syms x y t
# u1 = exp(-t)*(0.5-x)^2*(2-y)^2
# u2 = exp(-t)*(0.5-x^2)^2*(2-sin(y))^2

u1 = exp(-t)*(2-x)*x*(1-y)
u2 = exp(-t)*(2-x)*x*(1-y)
u11 = diff(u1, x)
u12 = diff(u1, y)
u21 = diff(u2, x)
u22 = diff(u2, y)

ε = [u11; u22; u12+u21]

E = 1.0
ν = 0.35
H = E/(1+ν)/(1-2ν)*[
  1-ν ν 0
  ν 1-ν 0
  0 0 (1-2ν)/2
]

σ = H * ε

σ = [σ[1] σ[3];σ[3] σ[2]]

f1 = u1 - (diff(σ[1,1], x) + diff(σ[1,2], y))
f2 = u2 - (diff(σ[2,1], x) + diff(σ[2,2], y))

s1 = sympy.julia_code(f1)
s1 = replace(replace(s1, ".^"=>"^"), ".*"=>"*")

s2 = sympy.julia_code(f2)
s2 = replace(replace(s2, ".^"=>"^"), ".*"=>"*")

println(s1)

println(s2)


function sieve(limit)
    primes = fill(true, limit)
    primes[1] = false
    for p in 2:limit
        primes[p] || continue
        for i = 2:div(limit, p)
            primes[p*i] = false
        end
    end
    find(primes)
end

println(sum(sieve(2_000_000)))

010 - Summation of Primes/010 - Summation_of_Primes.jl


using GridPotentialFlow
Δx = 0.01
Lx = 2.0
xlim = (-Lx/2,Lx/2)
ylim = (-Lx/2,Lx/2)
g = PhysicalGrid(xlim,ylim,Δx);

v = Vortex(0.0,0.0,1.0);

model = VortexModel(g,vortices=[v]);

s = streamfunction(model);
using Plots
plot(s,g,xlabel="x",ylabel="y")

q = curl(s);
plot(q,g,xlabel="x",ylabel="y")

function ψ_vortex!(ψ::Nodes{Dual,nx,ny},vortex::Vortex,g::PhysicalGrid) where {nx,ny}
    x,y = coordinates(ψ,g)
    for i in 2:nx-1, j in 2:ny-1
        r = sqrt((x[i]-vortex.x)^2+(y[j]-vortex.y)^2)
        ψ[i,j] = ψ_vortex(r,vortex.Γ)
    end
end

function ψ_vortex(r::Real,Γ::Real)
    return -Γ/(2π)*log(r)
end

nv = 4;
vl = Vortex.(-Lx/4 .+ 0.4*Lx*(rand(nv).-0.5),-Lx/4 .+ 0.4*Lx*(rand(nv).-0.5),0.5*rand(nv).+0.5);

grids = [PhysicalGrid(xlim,ylim,Lx/(nx-2)) for nx in [2^p for p in 5:9]];

using LinearAlgebra: norm

errors = []
gridspacings = []
for g in grids
    # LGF
    local model = VortexModel(g,vortices=vl)
    s_lgf = streamfunction(model);

    # Exact solution
    s_exact = Nodes(Dual,size(g))
    s_temp = Nodes(Dual,size(g))
    for v in vl
        ψ_vortex!(s_temp,v,g)
        s_exact += s_temp
    end

    # Bring to same reference level. A constant value can be added/subtracted to any potential flow solution
    s_lgf .-= s_lgf[g.I0[1],2];
    s_exact .-= s_exact[g.I0[1],2];

    error = s_lgf-s_exact
    idx = g.I0[1] + 1 : g.N[1] - 1 # Only look at top right corner
    push!(errors,norm(error[idx,idx])/norm(s_lgf[idx,idx]))
    push!(gridspacings,g.Δx)
end

firstorder=(errors[end]/gridspacings[end].*gridspacings).^1
secondorder=(sqrt(errors[end])/gridspacings[end].*gridspacings).^2
p=plot(gridspacings,errors,xaxis=:log,yaxis=:log,marker=:circle,lab="error",xlabel="dx",legend=:bottomright,title="Error")
plot!(gridspacings,firstorder,lab="1st order",linestyle=:dot,linecolor=:black)
plot!(gridspacings,secondorder,lab="2nd order",linestyle=:dash,linecolor=:black)

d = Lx/2
Γ = 1
v1 = Vortex(d/2,0.0,Γ);
v2 = Vortex(-d/2,0.0,Γ);

Vθ = Γ/(2*π*d); # Analytical tangential velocity
T = π*d/Vθ # Analytical period

model = VortexModel(g,vortices=[v1,v2]);

tspan = (0.0,T);

import OrdinaryDiffEq

#= To construct the `ODEProblem` from this package, we will have to provide a right-hand side function that computes the velocities $Ẋ$ of the vortices, which is the local flow velocity at their positions. These velocities are obtained using `vortexvelocities`, which regularizes the vorticity to the grid, solves the potential flow system, and interpolates the velocities from the grid to the vortex locations as

$\left(U_{q}, V_{q}\right)=\sum_{i, j} \mathsf{v}_{i j} d\left(\frac{\mathsf{x}_{i}-X_{q}}{\Delta x}\right)\left(\frac{\mathsf{y}_{j}-Y_{q}}{\Delta x}\right),$

where $v$ is the velocity field on the nodes.
=#

function rhs(X,model,t)
    setvortexpositions!(model,X)
    Ẋ = vortexvelocities!(model)
    return Ẋ
end

X = getvortexpositions(model)
prob = OrdinaryDiffEq.ODEProblem(rhs,X,tspan,model);
sol = OrdinaryDiffEq.solve(prob,dt=0.1,OrdinaryDiffEq.RK4(),dense=false,adaptive=false);

plot(sol.t,map(s->s.u[1],sol.u),xlabel='t',label="x₁")
plot!(sol.t,map(s->s.v[1],sol.u),label="y₁")

@testset "Corotating point vortices" begin
    @test isapprox(sol.u[end].u[1], 0.5; atol = 1e-2)
    @test isapprox(sol.u[end].u[2], -0.5; atol = 1e-2)
    @test isapprox(sol.u[end].v[1], 0.0; atol = 1e-2)
    @test isapprox(sol.u[end].v[2], 0.0; atol = 1e-2)
end

xlim = (-2,2);
ylim = (-2,2);
Δx = 0.05;
g = PhysicalGrid(xlim,ylim,Δx);

function vortexpatch!(vort,xc,yc,Γ,radius,nring)
    Δr = radius/(nring-1/2)
    dΓ = Γ/(1+8*nring*(nring-1)/2)
    push!(vort,Vortex(xc,yc,dΓ))
    for ir in 1:nring-1
        nθ = 8*ir
        for j = 0:nθ-1
            push!(vort,Vortex(xc + ir*Δr*cos(2π*j/nθ),yc + ir*Δr*sin(2π*j/nθ),dΓ))
        end
    end
    return vort
end

vortexpatch(xc,yc,Γ,radius,nring) = vortexpatch!(Vortex[],xc,yc,Γ,radius,nring)

d0 = d
anims = []
for r0 in [0.2*d0, 0.4*d0]
    vortices = vcat(vortexpatch(0.0,0.0+d0/2,Γ,r0,10),vortexpatch(0.0,0.0-d0/2,Γ,r0,10));
    vortexcolors = vcat(fill(:blue,length(vortices)÷2),fill(:red,length(vortices)÷2));
    local model = VortexModel(g,vortices=vortices);
    local X = getvortexpositions(model)
    local prob = OrdinaryDiffEq.ODEProblem(rhs,X,tspan,model);
    local sol = OrdinaryDiffEq.solve(prob,dt=0.1,OrdinaryDiffEq.RK4(),dense=false,adaptive=false);
    anim = @animate for i=1:length(sol.t)-1
        plot(xlims=xlim,ylims=ylim,ratio=:equal,legend=:none,title="r0/d0 = $(r0/d0)")
        scatter!(sol.u[i].u,sol.u[i].v,markerstrokewidth=0,markersize=3,color=vortexcolors)
    end
    push!(anims,anim)
end

gif(anims[1])

gif(anims[2])

# This file was generated using Literate.jl, https://github.com/fredrikekre/Literate.jl


#
# Generate an initial point in a grid, with a minimum distance defined
#

# To plot:
# using Plots
# scatter([ p[i][1] for i in 1:N ],[ p[i][2] for i in 1:N ],markersize=0.5,label="")

import Random

# Function that introduces a perturbation in the grid point
perturbation(step,σ) = -σ + 2*σ*rand()

"""

```
initial_point(N,side,tol;seed=123)
```

Generates a grid of N particles in a square of side `side`, with
perturbed coordinates but which satisfy a minimum distance greater
than `tol`

`N`: number of particles.

`side`: side of the square

`tol`: minimum distance allowed between particles

The optional keyword parameter `seed`, if set to `-1`, produces a
random seed at each run.

## Example

Generate 10_000 points in a grid of side 100, with a guaranteed minimum
distance greater than 0.5:

```
p = initial_point(10_000,100,0.5)
```

Use a random seed:

```
p = initial_point(10_000,100,0.5,seed=-1)
```

"""
function initial_point(N,side,tol;seed=123)

  # Use a different seed if desired
  if seed > 0
    Random.seed!(seed)
  else
    Random.seed!()
  end
  p = [ zeros(2) for i in 1:N ]

  # Properties of the regular grid
  l = ceil(Int,sqrt(N))
  step = (side-tol)/l
  if tol > step
    error("tol must be smaller than (side-tol)/√N = $step")
  end

  # The grid has l^2 points, but this is generally greater
  # than N. Thus, we generate list of points to be skiped
  skip = Int[]
  while length(skip) < l^2 - N
    i = rand(1:l^2)
    if findfirst(isequal(i), skip) == nothing
      push!(skip,i)
    end
  end

  # Maximum perturbation
  σ = (step-tol)/2

  # Generate points
  ip = 0
  igrid = 0
  x = [ tol/2, tol/2 ]
  for i in 1:l
    for j in 1:l
      igrid += 1
      if findfirst(isequal(igrid),skip) != nothing
        continue
      end
      # update x, with a perturbation of ±0.1*step
      ip += 1
      p[ip] .= x .+ perturbation(step,σ)
      x[2] = x[2] + step
    end
    x[2] = tol/2
    x[1] = x[1] + step
  end

  return p
end

# From data
initial_point(data::Data;tol=0.9,seed=123) = initial_point(data.N,data.side,tol,seed=seed)


using Flux, DiffEqFlux, DifferentialEquations, Plots

## Setup ODE to optimize
function lotka_volterra(du,u,p,t)
  x, y = u
  α, β, δ, γ = p
  du[1] = dx = α*x - β*x*y
  du[2] = dy = -δ*y + γ*x*y
end
u0 = [1.0,1.0]
tspan = (0.0,10.0)
p = [1.5,1.0,3.0,1.0] #True Params
prob = ODEProblem(lotka_volterra,u0,tspan,p)

# Verify ODE solution
sol = solve(prob,Tsit5())
plot(sol)

# Generate data from the ODE
data_sol = solve(prob,Tsit5(),saveat=0.1)
A1 = data_sol[1,:] # length 101 vector
A2 = data_sol[2,:] # length 101 vector
t = 0:0.1:10.0
scatter!(t,A1,color=[1],label = "rabbits")
scatter!(t,A2,color=[2],label = "wolves")

# Build a neural network that sets the cost as the difference from the
# generated data and true data

p = param([4., 1.0, 2.0, 0.4]) # Initial Parameter Vector
function predict_rd() # Our 1-layer neural network
  diffeq_rd(p,prob,Tsit5(),saveat=0.1)
end
loss_rd() = sum(abs2,predict_rd()-data_sol) # loss function

# Optimize the parameters so the ODE's solution stays near 1

data = Iterators.repeated((), 100)
opt = ADAM(0.1)
cb = function () #callback function to observe training
  #= display(loss_rd()) =#
  # using `remake` to re-create our `prob` with current parameters `p`
  scatter(t,A1,color=[1],label = "rabbit data")
  scatter!(t,A2,color=[2],label = "wolves data")
  display(plot!(solve(remake(prob,p=Flux.data(p)),Tsit5(),saveat=0.1),ylim=(0,6),labels = ["rabbit model","wolf model"],color=[1 2]))
end
# Display the ODE with the initial parameter values.
cb()
Flux.train!(loss_rd, [p], data, opt, cb = cb)
p

# ----- Testing with proposed objective function -----

p = param([4., 1.0, 2.0, 0.4]) # Initial Parameter Vector
data_sim = reduce(hcat, data_sol.u)
function loss_rd() # Our 1-layer neural network
  sum(abs2.(adams_moulton_estimator(p, data_sim, t, lotka_volterra)))
end

# Optimize the parameters so the ODE's solution stays near 1

data = Iterators.repeated((), 100)
opt = ADAM(0.1)
cb = function () #callback function to observe training
  #= display(loss_rd()) =#
  # using `remake` to re-create our `prob` with current parameters `p`
  scatter(t,A1,color=[1],label = "rabbit data")
  scatter!(t,A2,color=[2],label = "wolves data")
  display(plot!(solve(remake(prob,p=Flux.data(p)),Tsit5(),saveat=0.1),ylim=(0,6),labels = ["rabbit model","wolf model"],color=[1 2]))
end
# Display the ODE with the initial parameter values.
cb()
Flux.train!(loss_rd, [p], data, opt, cb = cb)

# ----- Testing with Floudas' dataset -----

i = 1
prob = problem_set[i]
f = prob.fun
exp_data = prob.data
lb = prob.bounds[1]
ub = prob.bounds[end]
true_par = prob.phi
t = prob.t
shared_rand = rand(1:1234)
rng = MersenneTwister(shared_rand)
p_am = param([rand(rng, Uniform(lb[i], ub[i])) for i in 1:length(true_par)])
rng = MersenneTwister(shared_rand)
p_rd = param([rand(rng, Uniform(lb[i], ub[i])) for i in 1:length(true_par)])
println(p_am)
println(p_rd)

function loss_am() # Our 1-layer neural network
  sum(abs2.(adams_moulton_estimator(p_am, exp_data, t, f)))
end

# Optimize the parameters so the ODE's solution stays near 1

data = Iterators.repeated((), 1000)
opt_method = ADAM(2.0)
u0 = exp_data[:,1]
tspan = (t[1], t[end])
prob_instance = ODEProblem(f, u0, tspan)

function predict_rd() # Our 1-layer neural network
  diffeq_rd(p_rd,prob_instance,Tsit5(),saveat=t)
end
loss_rd() = sum(soft_l1.(abs2.((predict_rd()-exp_data)))) # loss function

cb_am = function () #callback function to observe training
  # @show(loss_am())
  # using `remake` to re-create our `prob` with current parameters `p`
  plot_canvas = plot()
  for i in 1:length(u0)
    scatter!(plot_canvas,t,transpose(exp_data[i,:]), color=[i], label="[A_$i]")
  end
  display(plot!(plot_canvas,solve(remake(prob_instance,p=Flux.data(p_am)),Tsit5(),saveat=t)))
end
# Display the ODE with the initial parameter values.
cb_am()
Flux.train!(loss_am, [p_am], data, opt_method, cb = cb_am)
println("True:\n$true_par\nEst:\n$p_am\nDistance:\n$(euclidean(true_par,p_am))")

cb_rd = function () #callback function to observe training
  # @show(loss_rd())
  # using `remake` to re-create our `prob` with current parameters `p`
  plot_canvas = plot()
  for i in 1:length(u0)
    scatter!(plot_canvas,t,transpose(exp_data[i,:]), color=[i], label="[A_$i]")
  end
  display(plot!(plot_canvas,solve(remake(prob_instance,p=Flux.data(p_rd)),Tsit5(),saveat=t)))
end
# Display the ODE with the initial parameter values.
cb_rd()
Flux.train!(loss_rd, [p_rd], data, opt_method, cb = cb_rd)

println("True:\n$true_par\nEst:\n$p_rd\nDistance:\n$(euclidean(true_par,p_rd))")
m = Chain(
          p_rd -> diffeq_rd(p_rd, prob_instance, Tsit5(), saveat=t),
          )
ps = Flux.params(m)
Flux.train!(loss_rd, ps, data, opt_method)

true_par
p_rd

# ----- Second example -----
u0 = Float32[2.; 0.]
datasize = 30
tspan = (0.0f0,1.5f0)

function trueODEfunc(du,u,p,t)
    true_A = [-0.1 2.0; -2.0 -0.1]
    du .= ((u.^3)'true_A)'
end

function ODEfunc(du,u,p,t)
    du .= ((u.^3)'*p)'
end

t = range(tspan[1],tspan[2],length=datasize)
prob = ODEProblem(trueODEfunc,u0,tspan)
ode_data = Array(solve(prob,Tsit5(),saveat=t))
plot(transpose(ode_data))

par = [param(1.0) param(1.0); param(1.0) param(1.0)]
#par = param([1. 1.; 1. 1.])
function f1(x)
  data_shooting_estimator_node(par, x, t, ODEfunc; steps=1)
end
function f2(x)
  data_shooting_estimator_node(par, x, t, ODEfunc; steps=2)
end

function fe(x)
    sum(abs2.(data_shooting_estimator_endnode(par, x, t, ODEfunc; steps=3)))
end

dudt = Chain(x -> f1(x), x -> f2(x))
ps = Flux.params(dudt)

pred = n_ode(ode_data) # Get the prediction using the correct initial condition
scatter(t,ode_data[1,:],label="data")
scatter!(t,Flux.data(pred[1,:]),label="prediction")
function predict_n_ode()
  n_ode(u0)
end
loss_n_ode() = sum(abs2,ode_data .- predict_n_ode())

data = Iterators.repeated((), 100)
opt_method = ADAM(0.1)
cb = function () #callback function to observe training
  display(loss_n_ode())
  # plot current prediction against data
  cur_pred = Flux.data(predict_n_ode())
  pl = scatter(t,ode_data[1,:],label="data")
  scatter!(pl,t,cur_pred[1,:],label="prediction")
  display(plot(pl))
end

# Display the ODE with the initial parameter values.
cb()

dudt(ode_data)
loss_testing() = sum(abs2, dudt(ode_data).-ode_data)
Flux.train!(loss_testing, par, data, opt_method)
params(dudt)
par

# ----- Try simples example -----

# Parameter that I want to learn
learn_p = param([4.01;1.1])
# Function used for generating output using learn_p
function f_square_cube(x)
    x.^learn_p
end
# Single layer neural network
model_chain = Chain(x -> f_square_cube(x),
                    x -> f_square_cube(x)
                    )
tracking = params(learn_p)
loss(x,y) = sum(abs2, model_chain(x) - y)
rand_data = rand((1.0:0.1:100.0),2,300)
square_cube_rand = rand_data.^[4;9]
data = Iterators.repeated((rand_data, square_cube_rand), 2000)
Flux.train!(loss, tracking, data, ADAM(0.1))
tracking

#----- Now to ODEs -----
i = 1
p_solve = problem_set[i]
ode_fun = p_solve.fun
phi = p_solve.phi
t = p_solve.t
ode_data = p_solve.data
learn_p = param([rand(1:100) for i in 1:length(phi)])
learn_p
t = range(t[1], stop=t[end], length=10)
tspan = (t[1], t[end])
ini_cond = p_solve.data[:,1]
ode_prob = ODEProblem(p_solve.fun, ini_cond, tspan, p_solve.phi)
ode_sol  = solve(ode_prob, lsoda(), saveat=reduce(vcat, t))
data_original = reduce(hcat, ode_sol.u)
ode_data = copy(data_original)

function run_for_steps(steps, learn_p)
    function f_learn(x)
        data_shooting_estimator_node(learn_p,x,t,ode_fun; steps=steps)
    end
    # Single layer neural network
    model_chain = Chain(x -> f_learn(x))

    tracking = params(learn_p)
    function loss(x,y)
        res = Flux.mse(model_chain(x), y)
        if res == NaN
            res = 10^5
        end
        return res
    end
    data = Iterators.repeated((ode_data, ode_data), 500)
    Flux.train!(loss, tracking, data, ADAM(0.1))
    tracking
end

@time run_for_steps(1, learn_p)
run_for_steps(2, learn_p)
run_for_steps(3, learn_p)
run_for_steps(40, learn_p)

run_for_steps(40, learn_p)

# One step only
# Function used for generating output using learn_p
learn_p = param([7.71, 8.81])
function f_learn(x)
    data_shooting_estimator_node(learn_p,x,t,ode_fun; steps=1)
end
# Single layer neural network
model_chain = Chain(x -> f_learn(x))

tracking = params(learn_p)
loss(x,y) = sum(abs2, model_chain(x) - y)
data = Iterators.repeated((ode_data, ode_data), 10000)
Flux.train!(loss, tracking, data, ADAM(0.1))
tracking

# ----- Testing Adams-Moulton -----
i = 6
p_solve = problem_set[i]
ode_fun = p_solve.fun
phi = p_solve.phi
t = p_solve.t
ode_data = p_solve.data
t = p_solve.t
t = range(t[1], stop=t[end], length=50)
tspan = (t[1], t[end])
ini_cond = p_solve.data[:,1]
ode_prob = ODEProblem(p_solve.fun, ini_cond, tspan, p_solve.phi)
ode_sol  = solve(ode_prob, lsoda(), saveat=reduce(vcat, t))
data_original = reduce(hcat, ode_sol.u)
ode_data = copy(data_original)
plot(transpose(ode_data))
learn_p = param([rand(0.0001:0.0001:100) for i in 1:length(phi)])
tracking = params(learn_p)
function f_learn(x)
    adams_moulton_fourth_estimator(learn_p,x,t,ode_fun; return_estimated=true)
end
# Single layer neural network
model_chain = Chain(x -> f_learn(x))
function loss_mse(x,y)
    res = sum(abs2, model_chain(x) - y) + 10^10
    return res
end
data = Iterators.repeated((ode_data, ode_data), 1000)
function_error = []
function cb()
    err = Tracker.data(loss_mse(ode_data, ode_data))
    if err < 100
        push!(function_error, err)
    end
end
@time Flux.train!(loss_mse, tracking, data, ADAM(0.1), cb = cb)
plot(function_error[2:end])
tracking
euclidean(learn_p, phi)

# ----- Adams Moulton One on Each Neuron -----
i = 7
p_solve = problem_set[i]
ode_fun = p_solve.fun
phi = p_solve.phi
t = p_solve.t
ode_data = p_solve.data
t = p_solve.t
t = range(t[1], stop=t[end], length=50)
tspan = (t[1], t[end])
ini_cond = p_solve.data[:,1]
ode_prob = ODEProblem(p_solve.fun, ini_cond, tspan, p_solve.phi)
ode_sol  = solve(ode_prob, lsoda(), saveat=reduce(vcat, t))
data_original = reduce(hcat, ode_sol.u)
ode_data = copy(data_original)
plot(transpose(ode_data))
learn_p = param([rand(0.0001:0.0001:100) for i in 1:length(phi)])
tracking = params(learn_p)
function f_learn(x)
    adams_moulton_estimator(learn_p,x,t,ode_fun; return_estimated=true)
end
# Single layer neural network
function concat_neurons(args...)
    reestructure = zeros(eltype(args[1]), length(args[1][:,end]), length(args))
    for i in 1:length(args)
        reestructure[:,i] = args[i][:,end]
    end
    reestructure
end
a1 = adams_moulton_estimator(phi, ode_data[:,1:2], t[1:2], ode_fun; return_estimated=true)
a2 = adams_moulton_estimator(phi, ode_data[:,2:3], t[2:3], ode_fun; return_estimated=true)
a3 = adams_moulton_estimator(phi, ode_data[:,3:4], t[3:4], ode_fun; return_estimated=true)
con = concat_neurons(a1, a2, a3)
ode_data[:,2:4]
con - ode_data[:,2:4]

model_chain = Chain(x -> f_learn(x), x -> concat_neurons(x))
function loss_mse(x,y)
    res = Flux.mse(model_chain(x), y)
end

function loss_mse()
    proc_data = [ode_data[:,i:i+1] for i in 1:size(ode_data)[2]-1]
    segment = [f_learn(data_piece) for data_piece in proc_data]
    estimated_data = concat_neurons(segment...)
    res = Flux.mse(estimated_data, ode_data[:,2:end])
end

#data = Iterators.repeated([(ode_data[:,i:i+1], ode_data[:,2:end]) for i in 1:size(ode_data)[2]-1], 10000)
#data = [(ode_data[:,i:i+1], ode_data[:,2:end]) for i in 1:size(ode_data)[2]-1]
data = Iterators.repeated((), 1000)
function_error = []
function cb()
    err = Tracker.data(loss(ode_data, ode_data))
    if err < 100
        push!(function_error, err)
    end
end
@time Flux.train!(loss_mse, tracking, data, ADAM(0.1))
tracking
euclidean(learn_p, phi)
phi

# ----- Another test, adding layers  -----
i = 7
p_solve = problem_set[i]
ode_fun = p_solve.fun
phi = p_solve.phi
t = p_solve.t
ode_data = p_solve.data
t = p_solve.t
t = range(t[1], stop=t[end], length=10)
tspan = (t[1], t[end])
ini_cond = p_solve.data[:,1]
ode_prob = ODEProblem(p_solve.fun, ini_cond, tspan, p_solve.phi)
ode_sol  = solve(ode_prob, lsoda(), saveat=reduce(vcat, t))
data_original = reduce(hcat, ode_sol.u)
ode_data = copy(data_original)
plot(transpose(ode_data))
rand_ini = [rand(0.0001:0.0001:100) for i in 1:length(phi)]
learn_p = param(rand_ini)
#learn_p = param(phi)
#tracking = params(learn_p)
function f_learn(p)
    vec(adams_moulton_estimator(p, ode_data, t, ode_fun; return_estimated=true))
end
# 10 layer neural network
size(ode_data)
model_chain = Chain(Dense(length(ode_data),length(phi), σ),
                    Dense(length(phi),100, σ),
                    Dense(100,100, σ),
                    Dense(100,100, σ),
                    Dense(100,100, σ),
                    Dense(100,100, σ),
                    Dense(100,100, σ),
                    Dense(100,length(phi), abs),
                    x -> f_learn(x))
tracking = params(model_chain)
function loss(x,y)
    res = Flux.mse(model_chain(x), y)
    if res == NaN
        res = 10^10
    end
    return res
end
data = Iterators.repeated((vec(ode_data), vec(ode_data)), 1000000)
function_error = []
function cb()
    println(Flux.data(loss(vec(ode_data), vec(ode_data))))
end
@time Flux.train!(loss, tracking, data, ADAM(0.1))
print(params(model_chain))
# ----- Simpler model -----
learn_p = param([7.71]) |> gpu
function f(x)
    x.*learn_p
end
# Single layer neural network
model_chain = Chain(x -> f(x)) |> gpu
tracking = params(learn_p)
loss(x,y) = sum(abs2, model_chain(x) - y)
rand_data = rand((1.0:0.1:100.0),500,500) |> gpu
label = rand_data.*2 |> gpu
data = Iterators.repeated((rand_data, label), 5000)
@time Flux.train!(loss, tracking, data, ADAM(0.1))
tracking

# ----- Other one -----
model_chain = Chain(Dense(100, 50, σ),
                    Dense(50,100), softmax) |> gpu
tracking = params(model_chain) |> gpu
loss(x,y) = sum(abs2, model_chain(x) - y)
rand_data = rand((1.0:0.1:100.0),100,100)
label = rand_data.*[2]
data = Iterators.repeated((rand_data, label), 500)
@time Flux.train!(loss, tracking, data, ADAM(0.1))
loss(rand_data, label)

# ----- CuArrays are fast as they get big -----
a = rand((1:0.001:1000),10000,10000)
b = cu(a)
@time a*a
@time b*b


using LinearAlgebra
A = [1 -2  2; 1 -1  2; -1  1  1];
det(A)

b = [5, 7, 5];
v = A\b

transpose(v)

A1 = A[:, 2:3]

(A1\b)'

A2 = A[1:2,:]; b2 = b[1:2];
(A2\b2)'

Alu = lu(A)

Alu.U # similar for the lower, i.e. Alu.L

Alu.L*Alu.U  # => A (original matrix)

Alu.P

#=
Suppose we have equations:

 x - 2y + 2z = 5
 x -  y + 2z = 3
-x +  y + z  = 6

=#

b = [5; 3; 6];

# Ax = b => LUx = b :  x = inv(U)*inv(L)*b

(x,y,z) = inv(Alu.U)*inv(Alu.L)*b

# Compute the eigenvalues of A
# (These are complex numbers)
U = eigvals(A)

# ... and the eigenvectors
V = eigvecs(A)

#=
The eigenvectors are the columns of the V matrix.
=#

A*V[:,1] - U[1]*V[:,1]

#=
That is, all the real and imaginary parts are of the e-16 order,
so this is in effect a zero matrix of complex numbers.
=#

A = [1 -2  2; 1 -1  2; -1  1  1];
Diagonal(diag(A))

diag(A)

n = 2000;
B = randn(n,n);
B1 = B + B';
B2 = copy(B1);
B2[1,2] += 1eps();
B2[2,1] += 2eps();
issymmetric(B1)'; # => true
issymmetric(B2)'; # => false

@time eigvals(B1);

@time eigvals(B2);

@time eigvals(Symmetric(B2));

# Signal Processing
using PyPlot

fq = 500.0;
N = 512;
T = 6 / fq;
t = collect(range(0, stop=T, length=N));

x1 = sin.(2π * fq * t);
x2 = cos.(8π * fq * t);
x3 = cos.(16π * fq * t);
x = x1 + 0.4*x2 + 0.2*x3;

PyPlot.plot(t,x)

## using Pkg; Pkg.add("FFTW")
using FFTW

X = rfft(x)
sr = N / T

fd = collect(range(0, stop = sr/2, length = div(N,2) + 1))
yy = abs.(X);

plot(fd[1:64], yy[1:64])

using DSP

ns = 0.1*randn(length(x));
xn = x + ns;
M = 16;
xm = ones(Float64, M) / M;
xf = conv(xn, xm)
plot(1:length(xf), xf)

responsetype = Lowpass(0.2)

prototype = Elliptic(4, 0.5, 30)
tf = convert(TFFilter, digitalfilter(responsetype, prototype))

numerator_coefs   = coefb(tf)
denominator_coefs = coefa(tf)

responsetype = Bandpass(10, 40; fs=1000)
prototype = Butterworth(4)

xb = filt(digitalfilter(responsetype, prototype), x)›
plot(1:length(xb), xb)

# Image Processing

img = open("Files/lena.pgm");
magic  = chomp(readline(img));
params = chomp(readline(img));
pm = split(params)

# Remember the GSD

try
  global wd = parse(Int64,pm[1]);
  global ht = parse(Int64,pm[2]);
catch
  error("Can't figure out the image dimensions")
end

# Version 1.0 way of defining a byte array
# readbytes!() will read in place

data = Array{UInt8,2}(undef,wd,ht)
readbytes!(img, data, wd*ht);

data = reshape(data,wd,ht);
close(img);

# Define a convolution mask

Gx = [1 2 1; 0 0 0; -1 -2 -1];
Gy = [1 0 -1; 2 0 -2; 1 0 -1];

dout = copy(data);
for i = 2:wd-1
  for j = 2:ht-1
    temp = data[i-1:i+1, j-1:j+1];
    x = sum(Gx.*temp)
    y = sum(Gy.*temp)
    p = Int64(floor(sqrt(x*x + y*y)))
    dout[i,j] = (p < 256) ? UInt8(p) : 0xff
  end
end

# ... and output the result
out = open("lenaX.pgm","w");
println(out,magic);
println(out,params);
write(out,dout);
close(out);

# This only works if you have Imagemagick (or similar) installed
run(`display lenaX.pgm`);

using Plots; gr()

# http://docs.juliadiffeq.org/latest/

#=
OrdinaryDiffEq.jl is part of the JuliaDiffEq common interface,
but can be used independently of DifferentialEquations.jl.

User passes to OrdinaryDiffEq.jl an algorithm to solve
=#

using OrdinaryDiffEq

function ff(d,u,p,t)
  d[1] =  u[1] - u[1]*u[2]
  d[2] = -u[2] + u[1]*u[2] - u[2]*u[3]
  d[3] = -u[3] + u[2]*u[3]
end

u0 = [0.5; 1.0; 2.0];    # Setup the initial conditions
tspan = (0.0,10.0);       # and the time range

#=
In OrdinaryDiffEq.jl, some good "go-to" choices for ODEs are:

AutoTsit5(Rosenbrock23()) handles both stiff and non-stiff equations. This is a good algorithm to use if you know nothing about the equation.
BS3() for fast low accuracy non-stiff.
Tsit5() for standard non-stiff. This is the first algorithm to try in most cases.
Vern7() for high accuracy non-stiff.
Rodas4() for stiff equations with Julia-defined types, events, etc.
radau() for really high accuracy stiff equations (requires installing ODEInterfaceDiffEq.jl)

=#

prob = ODEProblem(ff,u0,tspan)

u = solve(prob, Tsit5());

# Plot API will plot the array
styles = [:solid; :dash; :dot]
N = length(styles)
styles = reshape(styles, 1, N)  # styles is now a 1xN Vector

Plots.plot(u, line = (2,styles))

using Sundials
function exotherm(t, x, dx; n=1, a=1, b=1)
  p = x[2]^n * exp(x[1])
  dx[1] = p - a*x[1]
  dx[2] = -b*p
  return(dx)
end

t = collect(range(0.0; stop=5.0,length=1001))
fexo(t,x,dx) = exotherm(t, x, dx, a=0.6, b=0.1)
x1 = Sundials.cvode(fexo, [0.0, 1.0], t)

PyPlot.plot(x1[:,1])

using Roots, Printf

f(x,a) = exp(x) - a*x

for p = 2.8:-0.02:2.6
  try
    ff(x) = f(x,p)
    @printf "%.2f : %.5f\n" p find_zero(ff,1.0)
  catch
    error("No convergence for parameter value: $p")
  end
end

# There is no solution for a <= exp(1) => 2.7182 .....

function lorenz(du,u,p,t)
 du[1] = 10.0(u[2]-u[1])
 du[2] = u[1]*(28.0-u[3]) - u[2]
 du[3] = u[1]*u[2] - (8/3)*u[3]
end
u0 = [1.0;0.0;0.0]
tspan = (0.0,100.0)
prob = ODEProblem(lorenz,u0,tspan)
sol = solve(prob,CVODE_Adams())
Plots.plot(sol,vars=(1,2,3))

using DifferentialEquations

f(du,u,p,t) = (du .= u)
g(du,u,p,t) = (du .= u)

u0 = rand(4,2);
W = WienerProcess(0.0,0.0,0.0);

prob = SDEProblem(f,g,u0,(0.0,1.0),noise=W)
sol = solve(prob,SRIW1());

# using Plots
# gr()
Plots.plot(sol)

function f(du,u,p,t)
  du[1] = u[1]
end

prob = ODEProblem(f,[0.2],(0.0,10.0))

rate(u,p,t) = 2
affect!(integrator) = (integrator.u[1] = integrator.u[1]/2)
jump = ConstantRateJump(rate,affect!)
jump_prob = JumpProblem(prob,Direct(),jump)

sol = solve(jump_prob,Tsit5())
Plots.plot(sol)

using Calculus

f(x)=sin(x)*cos(x)
derivative(f,1.0)

# Check since d(f) => cos*cos - sin*sin
cos(1.0)^2 - sin(1.0)^2

# Possible to curry the function
df = derivative(f)
df(1.0)

# Also defined is the 2nd derivative
d2f = second_derivative(f)
d2f(1.0)

# Also can use tick notation forhigher derivatives
# e.g. Value of the 3rd derivative
f'''(1.0)

#
# There are 2D functions, argument is a N-vector
# Be careful of name clashes
#
h(x) = (1+x[1])*exp(x[1])*sin(x[2])*cos(x[2])
gd=Calculus.gradient(h)
gd([1.0,1.0])

hs = Calculus.hessian(h)
hs([1.0,1.0])

# It is possible to output the symbolic version of the derivate
differentiate("sin(x)*cos(x)", :x)

# Not that clear but can be simplfied somewhat
# ... although not entirely perfect.
simplify(differentiate("sin(x)*cos(x)", :x))

# These techniques work more than a single variable too
# We get a 2-D array of partial derivatives
# Clearly the terms with a '0' multiplier can be ignored
#
simplify(differentiate("x*exp(-x)*sin(y)", [:x, :y]))

struct D <: Number
  d1::Float64
  d2::Float64
end

import Base: +, /, convert, promote_rule
+(x::D, y::D) = D(x.d1+y.d1, x.d2+y.d2)
/(x::D, y::D) = D(x.d1/y.d1, (y.d1*x.d2 - x.d1*y.d2)/y.d1^2)
convert(::Type{D}, x::Real) = D(x,zero(x))
promote_rule(::Type{D}, ::Type{<:Number}) = D

da = D(17,1)

function hero(x; k::Integer = 10)
  @assert k > 0
  t = (1+x)/2
  for i = 2:k
    t = (t + x/t)/2
  end
  return t
end

Call the function with the D() structure.
db = hero(da)

db.d1^2   # Confirm the solution

How does this work?
Decode it without the 'dual' number

using Printf
function dhero(x; k = 10)
    t  = (1+x)/2
    dt = 1
    @printf "%3d : %.5f : %.5f\n" 1 t dt
    for i = 2:k;
        t  = (t+x/t)/2;
        dt = (dt + (t - x*dt)/t^2)/2;
        @printf "%3d : %.5f : %.5f\n" i t dt
    end
    (t,dt)
end

dhero(17, k=5)

using SymPy

Display Hero's first 4 terms
Gets a little complex after that, try upping M!

M = 4
x = symbols("x")
for i = 1:M
  display(simplify(hero(x, k = i )))
end

using QuadGK
f(x) = sin(x)*(1.0 + cos(x))
quadgk(f,0.0,1.0)

#=
Addition function gives points and weights over the interval [-1,1]
Pick a sigmodial function : 1 - x exp(-|x|)
=#
g(u) = 1 - u*exp(-abs(u))

using PyPlot
x = collect(-1.0:0.1:1.0)
y = g.(x)
PyPlot.plot(x,y)

# The gauss() function creates a tuple of N array of points and weights
(x,w) = gauss(20)
s = sum([w[i]*g(x[i]) for i = 1:20])

# Alternate package which will do multidimension quadratutes
# Written by Steven Johnson (of PyCall, IJulia, etc. )
# It also does 1-D integration

using HCubature
hquadrature(f,0.0,1.0)

# But the power is the n-D quadratures
# Notice that the function expects an array argument

h(x) = 2.0*x[1]*exp(-x[1])*sin(x[2])*cos(x[2])
hcubature(h, [0,0], [1,1])

# Add JuMP#master to get v0.19

using JuMP, Clp

#=
Maximize the 5x+3y function subject to the constraint that: 3x+5y < 7
=#
m = Model(with_optimizer(Clp.Optimizer))

@variable(m, 0 <= x <= 5 );
@variable(m, 0 <= y <= 10 );

@objective(m, Max, 5x + 3y );
@constraint(m, 2x + 5y <= 7.0 );

JuMP.optimize!(m)

println("x = ", JuMP.value(x), " y = ", JuMP.value(y))

JuMP.objective_value(m)

using JuMP, LinearAlgebra, Printf

N = 6;
m = Model()
@variable(m, x[1:N], Bin);      # Define array variable to hold results
profit = [ 5, 3, 2, 7, 4, 4 ];  # Profit vector of size N
weight = [ 2, 8, 4, 2, 5, 6 ];  # Weights vector of size

maxcap = 15;

@objective(m, Max, dot(profit, x));
@constraint(m, dot(weight, x) <= maxcap);

using GLPK
JuMP.optimize!(m, with_optimizer(GLPK.Optimizer))

println("Objective is : ", JuMP.objective_value(m))
println("\nSolution is :")

for i = 1:N
    print("\tx[$i] = ", JuMP.value(x[i]))
    println(", p[$i]/w[$i] = ", profit[i]/weight[i])
end

using Optim

rosenbrock(x) =  (1.0 - x[1])^2 + 100.0 * (x[2] - x[1]^2)^2
result = Optim.optimize(rosenbrock, zeros(2), BFGS())

# Look at some values across a diagonal (x[1] == x[2]) cut
#
using Printf
for x in 0.0:0.1:1.3
  @printf "%3.1f : %f\n" x rosenbrock([x,x])
end

using SimJulia, ResumableFunctions
using Distributions
using Printf, Random

NUM_CUSTOMERS = 16     # total number of customers generated
NUM_TELLERS   = 2      # number of servers
QUEUE_MAX     = 2      # Maximum size of queue

μ = 0.4                # service rate
λ = 0.9                # arrival rate

arrival_dist = Exponential(1/λ)  # interarrival time distriubtion
service_dist = Exponential(1/μ)  # service time distribution

seed = ccall((:clock,"libc"),Int32,())
Random.seed!(seed);

queue_length = 0;
queue_stack  = Array{Integer,1}(undef,0);

@resumable function visit(env::Environment,
                        teller::Resource,
                        id::Integer,
                        time_arrvl::Float64,
                        dist_serve::Distribution)

# customer arrives
# recall the crazy scoping rules, we can see the array but not the scalar
#
    global queue_length
    @yield timeout(env, time_arrvl)
    @printf "Customer %2d %15s : %.3f\n" id "arrives" now(env)
    if queue_length > 0
        push!(queue_stack,id)
        println("CHECK: Length of the queue is $queue_length")
    end
    queue_length += 1
# customer starts to be served
    @yield request(teller)
    queue_length -= 1
    @printf "Customer %2d %15s : %.3f\n" id "being served" now(env)
#  teller is busy
    @yield timeout(env, rand(dist_serve))
# customer leaves
    @yield release(teller)
    @printf "Customer %2d %15s : %.3f\n" id "leaves" now(env)
end

# initialize simulation <: environment
sim     = Simulation()

# initialize service resources
service = Resource(sim, NUM_TELLERS)

# initialize customers and set arrival time
# customers arrive randomly baed on Poisson distribution
arrival_time = 0.0
for i = 1:NUM_CUSTOMERS
    arrival_time += rand(arrival_dist)
    @process visit(sim, service, i, arrival_time, service_dist)
end

# run the simulation
run(sim)

# Check on which customers had to wait
queue_stack


"""
# Bucket Sort

Bucket Sort is a divide and conquer sorting algorithm.
It distributes elements into buckets and sort each bucket before merging all buckets
Buckets are arrays that each stores a different range of elements
Insertion sort is used to sort the buckets because its fast in small-sized arrays

## Bucket Calculation

number of buckets = length of array
size of buckets = (largest element - smallest element + 1) / length of array

## Bucket Sort Steps

1. Create an array of buckets
2. Add each elements into its respective bucket
3. Sort each non-empty bucket
4. Concatenate elements of non-empty buckets

## Example

1. Array of [3, 5, 2, 9] has 4 buckets
2. The size of buckets = (9 - 2 + 1) / 4 = 2
3. The range of elements can be assign to each bucket, x
    bucket 1: 2 <= x < 4
    bucket 2: 4 <= x < 6
    bucket 3: 6 <= x < 8
    bucket 4: 8 <= x <= 9
4. The distribution of elements in the buckets = [3, 2][5][][9]
5. After sorting the elements each bucket = [2, 3][5][][9]
6. Concatenate all elements = [2, 3, 5, 9]

# Example
```julia
BucketSort!([3, 5, 2, 9]) # returns [2, 3, 5, 9]
```

Contributed By:- [Ming Liang](https://github.com/DrakonDarquesse)
"""

function BucketSort!(arr::Vector{T}, l::Int=length(arr)) where T
    if l==0
        return
    end

    max = maximum(arr)
    min = minimum(arr)
    r = (max-min+1)/l
    buckets = Array{T, 1}[[] for a in 1:l]

    for i in arr
        push!(buckets[floor(Int, (i-min)/r)+1], i)
    end

    for bucket in buckets
        for i in 1:length(bucket)
            val = bucket[i]
            idx = i
            while idx > 1 && val < bucket[idx-1]
                bucket[idx] = bucket[idx-1]
                idx = idx - 1
            end

            bucket[idx] = val
        end
    end
    copy!(arr, vcat(buckets...))
end


# ------------------------------------------------------------------------------------------
# # Linear algebra in Julia
# Based on work by Andreas Noack Jensen
#
# ## Basic linalg ops
# ------------------------------------------------------------------------------------------

# ------------------------------------------------------------------------------------------
# First let's define a random matrix
# ------------------------------------------------------------------------------------------

A = rand(1:4,3,3)

# ------------------------------------------------------------------------------------------
# Define a vector of ones
# ------------------------------------------------------------------------------------------

x = fill(1.0, (3))

# ------------------------------------------------------------------------------------------
# Notice that $A$ has type Array{Int64,2} but $x$ has type Array{Float64,1}. Julia defines
# the aliases Vector{Type}=Array{Type,1} and Matrix{Type}=Array{Type,2}.
#
# Many of the basic operations are the same as in other languages
# #### Multiplication
# ------------------------------------------------------------------------------------------

b = A*x

# ------------------------------------------------------------------------------------------
# #### Transposition
# As in other languages `A'` is the conjugate transpose
# ------------------------------------------------------------------------------------------

A'

# ------------------------------------------------------------------------------------------
# and we can get the transpose with
# ------------------------------------------------------------------------------------------

transpose(A)

# ------------------------------------------------------------------------------------------
# #### Transposed multiplication
# Julia allows us to write this without *
# ------------------------------------------------------------------------------------------

A'A

# ------------------------------------------------------------------------------------------
# #### Solving linear systems
# The problem $Ax=b$ for ***square*** $A$ is solved by the \ function.
# ------------------------------------------------------------------------------------------

A\b

# ------------------------------------------------------------------------------------------
# ## Special Matrix Structures
#
# Matrix structure is very important in linear algebra. To see *how* important it is, let's
# work with a larger linear system. Use the LinearAlgebra standard package to get access to
# structured matrices:
# ------------------------------------------------------------------------------------------

using LinearAlgebra

n = 1000
A = randn(n,n);

# ------------------------------------------------------------------------------------------
# Julia can often infer special matrix structure
# ------------------------------------------------------------------------------------------

Asym = A + A'
issymmetric(Asym)

# ------------------------------------------------------------------------------------------
# but sometimes floating point error might get in the way.
# ------------------------------------------------------------------------------------------

Asym_noisy = copy(Asym)
Asym_noisy[1,2] += 5eps()

issymmetric(Asym_noisy)

# ------------------------------------------------------------------------------------------
# Luckily we can declare structure explicitly with, for example, `Diagonal`, `Triangular`,
# `Symmetric`, `Hermitian`, `Tridiagonal` and `SymTridiagonal`.
# ------------------------------------------------------------------------------------------

Asym_explicit = Symmetric(Asym_noisy);

# ------------------------------------------------------------------------------------------
# Let's compare how long it takes Julia to compute the eigenvalues of `Asym`, `Asym_noisy`,
# and `Asym_explicit`
# ------------------------------------------------------------------------------------------

@time eigvals(Asym);

@time eigvals(Asym_noisy);

@time eigvals(Asym_explicit);

# ------------------------------------------------------------------------------------------
# In this example, using `Symmetric()` on `Asym_noisy` made our calculations about `5x` more
# efficient :)
# ------------------------------------------------------------------------------------------

# ------------------------------------------------------------------------------------------
# #### A big problem
# Using the `Tridiagonal` and `SymTridiagonal` types to store tridiagonal matrices makes it
# possible to work with potentially very large tridiagonal problems. The following problem
# would not be possible to solve on a laptop if the matrix had to be stored as a (dense)
# `Matrix` type.
# ------------------------------------------------------------------------------------------

n = 1_000_000;
A = SymTridiagonal(randn(n), randn(n-1));
@time eigmax(A)


using DifferentialEquations, Plots; pyplot()

beta, delta, gamma = 0.25, 0.2, 0.1
initialInfect = 0.025
println("R0 = ", beta/gamma)

initX = [1-initialInfect, 0.0, initialInfect, 0.0]
tEnd = 100.0

RHS(x,parms,t) = [  -beta*x[1]*x[3],
                    beta*x[1]*x[3] - delta*x[2],
                    delta*x[2] - gamma*x[3],
                    gamma*x[3] ]

prob = ODEProblem(RHS, initX, (0,tEnd), 0)
sol = solve(prob)
println("Final infected proportion= ", sol.u[end][4])

plot(sol.t,((x)->x[1]).(sol.u),label = "Susceptible", c=:green)
plot!(sol.t,((x)->x[2]).(sol.u),label = "Exposed", c=:blue)
plot!(sol.t,((x)->x[3]).(sol.u),label = "Infected", c=:red)
plot!(sol.t,((x)->x[4]).(sol.u),label = "Removed", c=:yellow,
    xlabel = "Time", ylabel = "Proportion",legend = :top)


# -*- coding: utf-8 -*-
# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .jl
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.5.0
#   kernelspec:
#     display_name: Julia 1.4.2
#     language: julia
#     name: julia-1.4
# ---

# # Solving Scalar Equations
#
# - https://docs.sciml.ai/v5.0.0/tutorials/ode_example.html#Example-1-:-Solving-Scalar-Equations-1

using DifferentialEquations
using Plots

# ## simple form

# +
f(u,p,t) = 1.01*u
u0=1/2
tspan = (0.0,1.0)
prob = ODEProblem(f,u0,tspan)
sol = solve(prob,Tsit5(),reltol=1e-8,abstol=1e-8)

plot(sol,linewidth=5,title="Solution to the linear ODE with a thick line",
     xaxis="Time (t)",yaxis="u(t) (in μm)",label="My Thick Line!") # legend=false
plot!(sol.t, t->0.5*exp(1.01t),lw=3,ls=:dash,label="True Solution!")
# -

# ## a little generic form

# +
# function form and `du` argument is added
function f(du, u, p, t)
    du[1] = 1.01 * u[1]
end

u0 = [1 / 2] # 1/2 -> [1/2] scalar to arrays
tspan = (0.0, 1.0)
prob = ODEProblem(f, u0, tspan)
sol = solve(prob,Tsit5(),reltol=1e-8,abstol=1e-8)

plot(sol,linewidth=5,title="Solution to the linear ODE with a thick line",
     xaxis="Time (t)",yaxis="u(t) (in μm)",label="My Thick Line!") # legend=false
plot!(sol.t, t->0.5*exp(1.01t),lw=3,ls=:dash,label="True Solution!")
# -

# # Lotka-Voltera equation
#
# - Code and other topics is taken from https://julialang.org/blog/2019/01/fluxdiffeq/#what_is_the_neural_ordinary_differential_equation_ode
#
# $$
# \begin{align}
# x' &= \alpha x + \beta xy \\
# y' &= -\delta y + \gamma xy
# \end{align}
# $$

# +
using DifferentialEquations

# define Differenial Equation
function lotka_volterra(du, u, p, t)
    x, y = u
    α, β, δ, γ = p
    du[1] = α*x - β*x*y
    du[2] = -δ*y + γ*x*y
end

# set initial
u0 = [1.0, 1.0]

tspan = (0.0, 10.0)
p = [1.5, 1.0, 4.0, 1.0]

# set ODE solver
prob = ODEProblem(lotka_volterra, u0, tspan, p)

sol = solve(prob)
using Plots
plot(sol)


cd(homedir()*"/PacktPub/Chp10")
const Chp10 = pwd()

using Loess, Plots
gr()

x = 10 .* rand(50)
y = 0.1*x .* sin.(x) .+ 0.2 * rand(50)
model = loess(x, y);

u = collect(minimum(x):0.1:maximum(x));
v = Loess.predict(model, u);

p = scatter(x,y)
plot!(p,u,v)

using LossFunctions

L2DistLoss()

true_targets = [  1,  0, -2, 1, 1.5] ;
pred_outputs = [0.5,  2, -1, 1, 2];

value(L2DistLoss(), true_targets, pred_outputs)
loss = L2DistLoss()

hcat(value(loss, true_targets, pred_outputs),
     deriv(loss, true_targets, pred_outputs),
     deriv2(loss,true_targets, pred_outputs))'

A = rand(3,4)

B = rand(3,4)

W = [2,1,3,2];

v1 = value(L2DistLoss(), A, B, AvgMode.WeightedSum(W))
v2 = value(L2DistLoss(), A, B, AvgMode.WeightedMean(W))
round.([v1, v2], digits=4)

deriv(L2DistLoss(), A, B)'

deriv2(L2DistLoss(), A, B)'

using RDatasets, Clustering

# Fully qualify as Metalhead (below) as dataset function
iris = RDatasets.dataset("datasets", "iris")
iris[1:5,:]

features = Matrix(iris[:,[1,2,3,4]])'
result   = kmeans( features, 3)

fieldnames(KmeansResult)
# [length(features[i,:]) for i in 1:3]

length(result.assignments) == size(iris)[1]

result.iterations, result.converged, result.totalcost

result.assignments'

using Plots; gr()
scatter(features[1,:], features[2,:], features[4,:], color = result.assignments)

scatter(features[1,:], features[2,:], features[4,:], color = result.assignments)

results = [kmeans( features, i) for i = 2:6]
for i in 1:length(results)
    println(i+1, " => ", results[i].totalcost)
end

result2.iterations, result2.converged, result2.totalcost

labels = iris[:, 5];
plt = Gadfly.plot(x=Y[:,1], y=Y[:,2], color=labels)

using DecisionTree
using Random, Statistics

# Create a random dataset
Random.seed!(systime())
X = sort(5 * rand(80))
XX = reshape(X, 80, 1)
y = X .* sin.(X)
y[1:5:end] += 3 * (0.5 .- rand(16))
;

# Fit regression model
regr_1 = DecisionTreeRegressor()
regr_2 = DecisionTreeRegressor(pruning_purity_threshold=0.05)
regr_3 = RandomForestRegressor(n_trees=20)
DecisionTree.fit!(regr_1, XX, y)
DecisionTree.fit!(regr_2, XX, y)
DecisionTree.fit!(regr_3, XX, y)
;

# Predict
X_test = 0:0.01:5.0
y_1 = DecisionTree.predict(regr_1, hcat(X_test))
y_2 = DecisionTree.predict(regr_2, hcat(X_test))
y_3 = DecisionTree.predict(regr_3, hcat(X_test))
;

using Plots
gr()
Plots.scatter(X, y, label="data", color="lightblue", legend=:bottomleft)
Plots.plot!(X_test, y_1, color="black", label="no pruning", linewidth=3)
Plots.plot!(X_test, y_2, color="red", label="pruning threshold = 0.05", linewidth=3)
Plots.plot!(X_test, y_3, color="blue", label="Random Forest Classifier", linewidth=3)
Plots.title!("Decision Tree Regression")


### A Pluto.jl notebook ###
# v0.11.14

using Markdown
using InteractiveUtils

# ╔═╡ 4723a50e-f6d1-11ea-0b1c-3d33a9b92f87
md"# Defining new types"

# ╔═╡ b3bb0212-f6da-11ea-2bea-21a84ea741a9
md"In this notebook we will see how to define our own types in Julia, and why it's useful to do so."

# ╔═╡ bed2a154-f6d1-11ea-0812-1f5c628a9785
md"## Why define new types?"

# ╔═╡ c826737a-f6d1-11ea-188c-8b6435fcd351
md"Many mathematical and other objects can be represented by a pair of numbers, for example:

  - a rectangle has a width and a height;
  - a complex number has a real and imaginary part;
  - a position vector in 2D space has 2 components.

Each of these could naturally be represented by a pair $(x, y)$ of two numbers, or in Julia as a tuple:
"

# ╔═╡ 81113974-f6d2-11ea-2ef8-fb2930402a74
begin
	rectangle = (3, 4)   # (width, height)
	c = (3, 4)   # 3 + 4im
	x = (3, 4)   # position vector
end

# ╔═╡ ac321d80-f6d2-11ea-2951-676e6e1aef56
md"But from the fact that we have to remind ourselves using comments what each oof these numbers represents, and the fact that they all look the same but should behave very differently, that there is a problem here.

For example, we would like to have a function `width` that returns the width of a rectangle, but it makes no sense to apply that to a complex number.
"

# ╔═╡ 645503ce-f6d7-11ea-37a1-915ed24aefb8
md"In other words, we need a way to be able to distinguish between different *types of* objects with different *behaviours*."

# ╔═╡ 9951a870-f6d7-11ea-1d94-ad757a8b196f
md"## Defining new types"

# ╔═╡ 9f38efc6-f6d7-11ea-088c-c5934f8ed969
md"""Julia allows us to do this by **defining new types**. For example, we would like to define a new **type** of, or *kind* of, object, called a `Rectangle`. Any object of type `Rectangle` should have its own `width` and `height` **fields** that are separate from any other `Rectangle` object, and indeed from any variables lying around that happen to be called that.

We can think of a `Rectangle` object as a bag or box that contains two pieces of data, namely the values of that `Rectangle`'s `width` and `height` fields.

When we specify what a `Rectangle` is, we are actually specifying *what all `Rectangle` objects should look like*, i.e. a template or "cookie-cutter" shape for making new objects of that type.

The syntax in Julia to do this is as follows:
"""

# ╔═╡ 66abc04e-f6d8-11ea-27d8-9f8b14659755
struct Rectangle
	width::Float64
	height::Float64
end

# ╔═╡ 7571be3a-f6d8-11ea-174c-9d65d5185153
md"""The keyword `struct` (short for "structure") tells Julia that we are defining a new type. We list the field names together with **type annotations** using `::` that specify which type each field can contain."""

# ╔═╡ 9f384ac2-f6d8-11ea-297e-4bf09acf9fe7
md"Once Julia has this template, we can create objects which have that type as follows:"

# ╔═╡ b0dac516-f6d8-11ea-1bdb-b59723107206
r = Rectangle(1, 2.5)

# ╔═╡ b98b9faa-f6d8-11ea-3610-bf8a84af2b5a
md"We can check that `r` is now a variable whose type is `Rectangle`, in other words `r` *is a* `Rectangle`:"

# ╔═╡ cf1c4aae-f6d8-11ea-3200-c5fb458c7c09
typeof(r)

# ╔═╡ d0749974-f6d8-11ea-2f41-074b6744f3d5
r isa Rectangle

# ╔═╡ d372342c-f6d8-11ea-10cd-573cf7eab992
md"""We can  extract from `r` the information about the values of the fields that it contains using "`.`": """

# ╔═╡ dd8f9e88-f6d8-11ea-15e3-f17f4af0d81b
r.width

# ╔═╡ e3e70064-f6d8-11ea-22fd-892bbc567ed4
r.height

# ╔═╡ e582eb02-f6d8-11ea-1fcc-89bbc9dfbb07
md"We can create a new `Rectangle` with its own width and height:"

# ╔═╡ f2ed18b4-f6d8-11ea-3bc7-0b82eb5e8dc0
r2 = Rectangle(3.0, 4.0)

# ╔═╡ f9d192fe-f6d8-11ea-138d-3dcdff33c034
md"You should check that this does *not* affect the `width` and `height` variables belonging to `r`."

# ╔═╡ 6085144c-f6db-11ea-19fe-ed46dafb4562
md"Types like this are often called **composite types**; they consist of aggregating, or collecting together, different pieces of information that belong to a given object."

# ╔═╡ 1840898e-f6d9-11ea-3035-bb4dac496834
md"## Mutable vs immutable"

# ╔═╡ 299442a2-f6d9-11ea-1ed0-5fb7ccff2b35
md"Now suppose we want to change the width of `r`. We would naturally try the following:"

# ╔═╡ 63f76d28-f6d9-11ea-071c-458528c36008
r.width = 10

# ╔═╡ 68934a2a-f6d9-11ea-37ea-850304f6d3d6
md"But Julia complains that fields of objects of type `Rectangle` *cannot* be modified. This is because `struct` makes **immutable** (i.e. unchangeable) objects. The reason for this is that these usually lead to *faster code*."

# ╔═╡ a38a0164-f6d9-11ea-2e1d-a7a6e2106d0b
md"If we really want to have a **mutable** (modifiable) object, we can declare it using `mutable struct` instead of `struct` -- try it!"

# ╔═╡ 3fa7d8e2-f6d9-11ea-2e82-9f59b5cb9424
md"## Functions on types"

# ╔═╡ 87eeb3fe-f6da-11ea-2fa3-d95454cd50de
md"We can now define functions that act only on given types. To restrict a given function argument to only accept objects of a certain type, we add a type annotation:"

# ╔═╡ 8c4beb6e-f6db-11ea-12d1-cf450181363b
width(r::Rectangle) = r.width

# ╔═╡ 91e21d28-f6db-11ea-1b0f-336719682f28
width(r)

# ╔═╡ 2ef7392e-f6dc-11ea-00e4-770cdf9a102e
md"Applying the function to objects of other types gives an error:"

# ╔═╡ 9371209e-f6db-11ea-3ba2-c3597d42d8ed
width(3)   # throws an error

# ╔═╡ b916acb4-f6dc-11ea-3cdf-2b8ab3c34e03
md"""It is common in Julia to have "generic" versions of functions that apply to any object, and then specialised versions that apply only to certain types.

For example, we might want to define an `area` function with the correct definition for `Rectangle`s, but it might be convenient to fall back to a version that just returns the value itself for any other type:"""

# ╔═╡ 6fe1b332-f6dd-11ea-39d4-1954aeda6f08
begin
	area(r::Rectangle) = r.width * r.height
	area(x) = x
end

# ╔═╡ c34355b8-f6dd-11ea-1089-cf5be2117ba8
area(r)

# ╔═╡ c4a7fae6-f6dd-11ea-1851-3bd445ebf677
area(17)

# ╔═╡ c83180d8-f6dd-11ea-32f7-634b781070f1
md"But since we didn't restrict the type in the second method, we also have"

# ╔═╡ d3dc64b6-f6dd-11ea-1273-7fb3957e4964
area("hello")

# ╔═╡ 6d7ca590-f6f5-11ea-0a00-6128f971b546
area

# ╔═╡ d6e5a276-f6dd-11ea-34aa-b9e2d3805364
md"which does not make much sense."

# ╔═╡ 79fa562e-f6dd-11ea-2e97-df3c62c83685
md"Note that these are different versions of the function with the *same* function name; each version acting on different (combinations of) types is called a **method** of the (generic) function."

# ╔═╡ a1d2ae6c-f6dd-11ea-0216-ef9db5d9e29b
md"Suppose that later we create a `Circle` type. We can then just add a new method `area(c::Circle)` with the corresponding definition, and Julia will continue to choose the correct version (method) when we call `area(x)`, depending on the type of `x`.

[Note that at the time of writing, Pluto requires all methods of a function to be defined in the same cell.]
"

# ╔═╡ 41e21994-f6de-11ea-0e5c-0515a3a52f6f
md"## Multiple dispatch"

# ╔═╡ 46aed906-f6de-11ea-17d8-cb032f59240e
md"""The act of choosing which function to call based on the type of the arguments that are passed to the function is called **dispatch**. A central feature of Julia is **multiple dispatch**: this choice is made based on the types of *all* the arguments to a function.

For example, the following three calls to the `+` function each call *different* methods, defined in different locations. Click on the links to see the definition of each method in the Julia source code on GitHub!
"""

# ╔═╡ 814e328c-f6de-11ea-13c0-d1b97714c4f3
cc = 3 + 4im

# ╔═╡ cdd3e14e-f6f5-11ea-15e2-bd309e658823
cc + cc

# ╔═╡ e01e26f2-f6f5-11ea-13b0-95413a6f7290
+

# ╔═╡ 84cae75c-f6de-11ea-3cd4-1b263e34771f
@which cc + cc

# ╔═╡ 8ac4904a-f6de-11ea-105b-8925016ca6d5
@which cc + 3

# ╔═╡ 8cd9f438-f6de-11ea-2b58-93bbb860a005
@which 3 + cc

# ╔═╡ Cell order:
# ╟─4723a50e-f6d1-11ea-0b1c-3d33a9b92f87
# ╟─b3bb0212-f6da-11ea-2bea-21a84ea741a9
# ╟─bed2a154-f6d1-11ea-0812-1f5c628a9785
# ╟─c826737a-f6d1-11ea-188c-8b6435fcd351
# ╠═81113974-f6d2-11ea-2ef8-fb2930402a74
# ╟─ac321d80-f6d2-11ea-2951-676e6e1aef56
# ╟─645503ce-f6d7-11ea-37a1-915ed24aefb8
# ╟─9951a870-f6d7-11ea-1d94-ad757a8b196f
# ╟─9f38efc6-f6d7-11ea-088c-c5934f8ed969
# ╠═66abc04e-f6d8-11ea-27d8-9f8b14659755
# ╟─7571be3a-f6d8-11ea-174c-9d65d5185153
# ╟─9f384ac2-f6d8-11ea-297e-4bf09acf9fe7
# ╠═b0dac516-f6d8-11ea-1bdb-b59723107206
# ╟─b98b9faa-f6d8-11ea-3610-bf8a84af2b5a
# ╠═cf1c4aae-f6d8-11ea-3200-c5fb458c7c09
# ╠═d0749974-f6d8-11ea-2f41-074b6744f3d5
# ╟─d372342c-f6d8-11ea-10cd-573cf7eab992
# ╠═dd8f9e88-f6d8-11ea-15e3-f17f4af0d81b
# ╠═e3e70064-f6d8-11ea-22fd-892bbc567ed4
# ╟─e582eb02-f6d8-11ea-1fcc-89bbc9dfbb07
# ╠═f2ed18b4-f6d8-11ea-3bc7-0b82eb5e8dc0
# ╟─f9d192fe-f6d8-11ea-138d-3dcdff33c034
# ╟─6085144c-f6db-11ea-19fe-ed46dafb4562
# ╟─1840898e-f6d9-11ea-3035-bb4dac496834
# ╟─299442a2-f6d9-11ea-1ed0-5fb7ccff2b35
# ╠═63f76d28-f6d9-11ea-071c-458528c36008
# ╟─68934a2a-f6d9-11ea-37ea-850304f6d3d6
# ╟─a38a0164-f6d9-11ea-2e1d-a7a6e2106d0b
# ╟─3fa7d8e2-f6d9-11ea-2e82-9f59b5cb9424
# ╟─87eeb3fe-f6da-11ea-2fa3-d95454cd50de
# ╠═8c4beb6e-f6db-11ea-12d1-cf450181363b
# ╠═91e21d28-f6db-11ea-1b0f-336719682f28
# ╟─2ef7392e-f6dc-11ea-00e4-770cdf9a102e
# ╠═9371209e-f6db-11ea-3ba2-c3597d42d8ed
# ╟─b916acb4-f6dc-11ea-3cdf-2b8ab3c34e03
# ╠═6fe1b332-f6dd-11ea-39d4-1954aeda6f08
# ╠═c34355b8-f6dd-11ea-1089-cf5be2117ba8
# ╠═c4a7fae6-f6dd-11ea-1851-3bd445ebf677
# ╟─c83180d8-f6dd-11ea-32f7-634b781070f1
# ╠═d3dc64b6-f6dd-11ea-1273-7fb3957e4964
# ╠═6d7ca590-f6f5-11ea-0a00-6128f971b546
# ╟─d6e5a276-f6dd-11ea-34aa-b9e2d3805364
# ╟─79fa562e-f6dd-11ea-2e97-df3c62c83685
# ╟─a1d2ae6c-f6dd-11ea-0216-ef9db5d9e29b
# ╟─41e21994-f6de-11ea-0e5c-0515a3a52f6f
# ╟─46aed906-f6de-11ea-17d8-cb032f59240e
# ╠═814e328c-f6de-11ea-13c0-d1b97714c4f3
# ╠═cdd3e14e-f6f5-11ea-15e2-bd309e658823
# ╠═e01e26f2-f6f5-11ea-13b0-95413a6f7290
# ╠═84cae75c-f6de-11ea-3cd4-1b263e34771f
# ╠═8ac4904a-f6de-11ea-105b-8925016ca6d5
# ╠═8cd9f438-f6de-11ea-2b58-93bbb860a005