export linear, quadratic, quaddiff, cubic,
       neville, vandermonde, vandermonde!, cubichermite,
       LinearInterpolator, CubicInterpolator,
       BilinearInterpolator, BicubicInterpolator

"""
    linear(x, xₚ, yₚ)

Perform simple linear interpolation of the points defined by coordinates `xₚ` and values `yₚ`, at the coordinate `x`. `xₚ` and `yₚ` must both contain two points.
"""
function linear(x, xₚ, yₚ)
    @inbounds x₁, x₂ = xₚ[1], xₚ[2]
    @inbounds y₁, y₂ = yₚ[1], yₚ[2]
    return y₁ + (x - x₁)*(y₂ - y₁)/(x₂ - x₁)
end

"""
    quadratic(x, xₚ, yₚ)

Perform quadratic polynomial interpolation of the points defined by coordinates `xₚ` and values `yₚ`, at the coordinate `x`, using Neville's algorithm. `xₚ` and `yₚ` must both contain three points.
"""
function quadratic(x, xₚ, yₚ)
    @assert length(xₚ) == length(yₚ) == 3 "need 3 points for quadratic interpolation"
    #short names
    @inbounds x₁, x₂, x₃ = xₚ[1], xₚ[2], xₚ[3]
    @inbounds y₁, y₂, y₃ = yₚ[1], yₚ[2], yₚ[3]
    #first stage
    p₁₂ = ((x - x₂)*y₁ + (x₁ - x)*y₂)/(x₁ - x₂)
    p₂₃ = ((x - x₃)*y₂ + (x₂ - x)*y₃)/(x₂ - x₃)
    #final stage
    ((x - x₃)*p₁₂ + (x₁ - x)*p₂₃)/(x₁ - x₃)
end

"""
    cubic(x, xₚ, yₚ)

Perform cubic polynomial interpolation of the points defined by coordinates `xₚ` and values `yₚ`, at the coordinate `x`, using Neville's algorithm. `xₚ` and `yₚ` must both contain four points.
"""
function cubic(x, xₚ, yₚ)
    @assert length(xₚ) == length(yₚ) == 4 "need 4 points for cubic interpolation"
    #short names
    @inbounds x₁, x₂, x₃, x₄ = xₚ[1], xₚ[2], xₚ[3], xₚ[4]
    @inbounds y₁, y₂, y₃, y₄ = yₚ[1], yₚ[2], yₚ[3], yₚ[4]
    #first stage
    p₁₂ = ((x - x₂)*y₁ + (x₁ - x)*y₂)/(x₁ - x₂)
    p₂₃ = ((x - x₃)*y₂ + (x₂ - x)*y₃)/(x₂ - x₃)
    p₃₄ = ((x - x₄)*y₃ + (x₃ - x)*y₄)/(x₃ - x₄)
    #second stage
    p₁₂₃ = ((x - x₃)*p₁₂ + (x₁ - x)*p₂₃)/(x₁ - x₃)
    p₂₃₄ = ((x - x₄)*p₂₃ + (x₂ - x)*p₃₄)/(x₂ - x₄)
    #final stage
    ((x - x₄)*p₁₂₃ + (x₁ - x)*p₂₃₄)/(x₁ - x₄)
end

"""
    neville(x, xₚ, yₚ)

Perform polynomial interpolation of the points defined by coordinates `xₚ` and values `yₚ`, at the coordinate `x`, using Neville's algorithm with as many points as are provided. `xₚ` and `yₚ` must have the same length. With only 3 or 4 points the [`quadratic`](@ref) and [`cubic`](@ref) functions will be considerably faster.
"""
function neville(x, xₚ, yₚ)
    @assert length(xₚ) == length(yₚ) "can't Neville with vectors of different lengths"
    n = length(xₚ)
    P = zeros(n, n)
    P[:,1] = yₚ
    for i ∈ 2:n, j ∈ i:n
        @inbounds P[j,i] = ((x - xₚ[j-i+1])*P[j,i-1] - (x - xₚ[j])*P[j-1,i-1])/(xₚ[j] - xₚ[j-i+1])
    end
    return P[n,n]
end

"""
    vandermonde(x, y)

Generate the coefficients of an arbitrary order polynomial passing through the ponts defined by coordinates `x` and value `y`. For n points, n coefficients ``[c_0, c_1, ..., c_{n-1}]`` are returned forming the polynomial ``c_0 + c_1x + ... + c_{n-1}x^{n-1}``

!!! warning

    Solving for the the coefficients of a high-order polynomial is a notoriously ill-conditioned problem. It is not recommended for orders greater than 5 or 6, although it depends on the application. If you must interpolate with a high-order polynomial, it's better to use the [`neville`](@ref) function instead of computing coefficients.
"""
function vandermonde(x, y)
    @assert length(x) == length(y) "length of x must equal length of y"
    c = zeros(length(x))
    vandermonde!(c, x, y)
    return c
end

function vandermonde!(c, x, y)
    @assert length(c) == length(x) == length(y) "vandermonde! requires length(c) == length(x) == length(y)"
    n = length(x)
    s = similar(c)
    c[:] .= zero(eltype(c))
    s[n] = -x[1]
    @inbounds for i = 2:n
        for j = n-i+1:n-1
            s[j] -= x[i]*s[j+1]
        end
        s[n] -= x[i]
    end
    @inbounds for j = 1:n
        ϕ = n
        for k = n:-1:2
            ϕ = (k-1)*s[k] + x[j]*ϕ
        end
        f = y[j]/ϕ
        b = 1.0
        for k = n:-1:1
            c[k] += b*f
            b = s[k] + x[j]*b
        end
    end
end

"""
    cubichermite(x, x₁, x₂, y₁, y₂, y₁′, y₂′)

Interpolate a cubic polynomial between two points, given its values and first derivatives at the points.
"""
function cubichermite(x, x₁, x₂, y₁, y₂, y₁′, y₂′)
    Δx = (x₂ - x₁)
    ξ = (x - x₁)/Δx
    u = Δx*y₁′
    v = Δx*y₂′
    ξ^3*(2*y₁ + u - 2*y₂ + v) + ξ^2*(-3*y₁ - 2*u + 3*y₂ - v) + ξ*u + y₁
end

#-------------------------------------------------------------------------------
# simple piecewise linear interpolator

struct LinearInterpolator{T,B} <: OneDimensionalInterpolator
    r::InterpolatorRange{T}
    boundaries::B
    i::RefValue{Int64} #previous cell index
end

"""
    LinearInterpolator(x, y, boundaries=StrictBoundaries())

Construct a `LinearInterpolator` for the points defined by coordinates `x` and values `y`
"""
function LinearInterpolator(x, y, boundaries::AbstractBoundaries=StrictBoundaries())
    LinearInterpolator(InterpolatorRange(x, y), boundaries, Ref(1))
end

"""
    LinearInterpolator(f, xa, xb, n, boundaries=StrictBoundaries())

Construct a `LinearInterpolator` for the function `f` using `n` evenly spaced function evaluations in the range [`xa`,`xb`]
"""
function LinearInterpolator(f, xa, xb, n::Int, boundaries::AbstractBoundaries=StrictBoundaries())
    linstruct(LinearInterpolator, f, xa, xb, n, boundaries)
end

function (ϕ::LinearInterpolator)(x)
    #enforce boundaries
    ϕ.boundaries(x, ϕ.r.xa, ϕ.r.xb)
    #find the interpolation cell
    i = findcell(x, ϕ)
    #short names
    @inbounds x₁, x₂ = ϕ.r.x[i], ϕ.r.x[i+1]
    @inbounds y₁, y₂ = ϕ.r.y[i], ϕ.r.y[i+1]
    #interpolate
    (x - x₁)*(y₂ - y₁)/(x₂ - x₁) + y₁
end

Base.getindex(ϕ::LinearInterpolator, i) = ϕ.r.y[i]
Base.firstindex(::LinearInterpolator) = 1
Base.lastindex(ϕ::LinearInterpolator) = ϕ.r.n
function Base.setindex!(ϕ::LinearInterpolator, v, i)
    ϕ.r.y[i] = v
end
function Base.copy(ϕ::LinearInterpolator)
    LinearInterpolator(ϕ.r, ϕ.boundaries, Ref(1))
end

#-------------------------------------------------------------------------------
# piecewise cubic interpolator without continuous derivatives (not splines)

struct CubicInterpolator{T,B} <: OneDimensionalInterpolator
    r::InterpolatorRange{T}
    boundaries::B
    i::RefValue{Int64} #previous cell index
end

"""
    CubicInterpolator(x, y, boundaries=StrictBoundaries())

Construct a `CubicInterpolator` for the points defined by coordinates `x` and values `y`
"""
function CubicInterpolator(x, y, boundaries::AbstractBoundaries=StrictBoundaries())
    @assert length(x) > 3 "can't do cubic interpolation with < 4 points"
    CubicInterpolator(InterpolatorRange(x, y), boundaries, Ref(1))
end

"""
    CubicInterpolator(f, xa, xb, n, boundaries=StrictBoundaries())

Construct a `CubicInterpolator` for the function `f` using `n` evenly spaced function evaluations in the range [`xa`,`xb`]
"""
function CubicInterpolator(f, xa, xb, n::Int, boundaries::AbstractBoundaries=StrictBoundaries())
    linstruct(CubicInterpolator, f, xa, xb, n, boundaries)
end

function (ϕ::CubicInterpolator)(x)
    #enforce boundaries if desired
    ϕ.boundaries(x, ϕ.r.xa, ϕ.r.xb)
    #find the interpolation point
    i = findcell(x, ϕ)
    #determine which points to neville
    if i == 1
        I = 1:4
    elseif i == ϕ.r.n - 1
        I = ϕ.r.n-3:ϕ.r.n
    else
        I = i-1:i+2
    end
    #interpolate
    @inbounds cubic(x, view(ϕ.r.x,I), view(ϕ.r.y,I))
end

Base.getindex(ϕ::CubicInterpolator, i) = ϕ.r.y[i]
Base.firstindex(::CubicInterpolator) = 1
Base.lastindex(ϕ::CubicInterpolator) = ϕ.r.n
function Base.setindex!(ϕ::CubicInterpolator, v, i)
    ϕ.r.y[i] = v
end
function Base.copy(ϕ::CubicInterpolator)
    CubicInterpolator(ϕ.r, ϕ.boundaries, Ref(1))
end

#-------------------------------------------------------------------------------
# bilinear interpolator

struct BilinearInterpolator{T,B} <: TwoDimensionalInterpolator
    G::InterpolatorGrid{T}
    boundaries::B
    i::RefValue{Int64} #previous cell index
    j::RefValue{Int64} #previous cell index
end

"""
    BilinearInterpolator(x, y, Z, boundaries=StrictBoundaries())

Construct a `BilinearInterpolator` for the grid of points points defined by coordinates `x`,`y` and values `Z`.
"""
function BilinearInterpolator(x, y, Z, boundaries::AbstractBoundaries=StrictBoundaries())
    BilinearInterpolator(InterpolatorGrid(x, y, Z), boundaries, Ref(1), Ref(1))
end

"""
    BilinearInterpolator(f, xa, xb, nx, ya, yb, ny, boundaries=StrictBoundaries())

Construct a `BilinearInterpolator` for the function `f` using a grid of `nx` points evenly spaced on the first axis in [`xa`,`xb`] and `ny` points evenly spaced on the second axis in [`ya`,`yb`].
"""
function BilinearInterpolator(f,
                              xa, xb, nx::Int,
                              ya, yb, ny::Int,
                              boundaries::AbstractBoundaries=StrictBoundaries())
    linstruct(BilinearInterpolator, f, xa, xb, nx, ya, yb, ny, boundaries)
end

function (Φ::BilinearInterpolator)(x, y)
    #enforce boundaries if desired
    Φ.boundaries(x, Φ.G.xa, Φ.G.xb, y, Φ.G.ya, Φ.G.yb)
    #find the proper grid box to interpolate inside
    i, j = findcell(x, y, Φ)
    #clear names
    @inbounds x₁, x₂ = Φ.G.x[i], Φ.G.x[i+1]
    @inbounds y₁, y₂ = Φ.G.y[j], Φ.G.y[j+1]
    @inbounds Z₁₁, Z₂₁ = Φ.G.Z[i,j],     Φ.G.Z[i+1,j]
    @inbounds Z₂₂, Z₁₂ = Φ.G.Z[i+1,j+1], Φ.G.Z[i,j+1]
    #reused differences
    xx₁ = x - x₁
    x₂x₁ = x₂ - x₁
    #interpolate along axis 1 first
    t = xx₁*(Z₂₁ - Z₁₁)/x₂x₁ + Z₁₁
    u = xx₁*(Z₂₂ - Z₁₂)/x₂x₁ + Z₁₂
    #then finish by interpolating between the interpolated values along axis 2
    (y - y₁)*(u - t)/(y₂ - y₁) + t
end

Base.getindex(Φ::BilinearInterpolator, i, j) = Φ.G.Z[i,j]
function Base.setindex!(Φ::BilinearInterpolator, v, i, j)
    Φ.G.Z[i,j] = v
end
function Base.copy(Φ::BilinearInterpolator)
    BilinearInterpolator(Φ.G, Φ.boundaries, Ref(1), Ref(1))
end

#-------------------------------------------------------------------------------
# bicubic interpolator

struct BicubicInterpolator{T,B} <: TwoDimensionalInterpolator
    G::InterpolatorGrid{T}
    boundaries::B
    i::RefValue{Int64} #previous cell index
    j::RefValue{Int64} #previous cell index
end

"""
    BicubicInterpolator(x, y, Z, boundaries=StrictBoundaries())

Construct a `BicubicInterpolator` for the grid of points points defined by coordinates `x`,`y` and values `Z`.
"""
function BicubicInterpolator(x, y, Z, boundaries::AbstractBoundaries=StrictBoundaries())
    #insist on at least 4 points in each dimension
    @assert (length(x) > 3) & (length(y) > 3) "bicubic interpolation requires at least 4 points in each dimension"
    BicubicInterpolator(InterpolatorGrid(x, y, Z), boundaries, Ref(1), Ref(1))
end

"""
    BicubicInterpolator(f, xa, xb, nx, ya, yb, ny, boundaries=StrictBoundaries())

Construct a `BicubicInterpolator` for the function `f` using a grid of `nx` points evenly spaced on the first axis in [`xa`,`xb`] and `ny` points evenly spaced on the second axis in [`ya`,`yb`].
"""
function BicubicInterpolator(f,
                             xa, xb, nx::Int,
                             ya, yb, ny::Int,
                             boundaries::AbstractBoundaries=StrictBoundaries())
    linstruct(BicubicInterpolator, f, xa, xb, nx, ya, yb, ny, boundaries)
end

function (Φ::BicubicInterpolator)(x, y)
    #enforce boundaries if desired
    Φ.boundaries(x, Φ.G.xa, Φ.G.xb, y, Φ.G.ya, Φ.G.yb)
    #find the proper grid box to interpolate inside
    i, j = findcell(x, y, Φ)
    #get indices of points along axis 1 to use for interpolation
    if i == 1
        I = 1:4
    elseif i == Φ.G.nx - 1
        I = Φ.G.nx-3:Φ.G.nx
    else
        I = i-1:i+2
    end
    #get indices along axis 2 where initial 4 interpolations occur
    if j == 1
        J = 1:4
    elseif j == Φ.G.ny - 1
        J = Φ.G.ny-3:Φ.G.ny
    else
        J = j-1:j+2
    end
    #view the proper chunks of the arrays
    xᵣ = view(Φ.G.x, I)
    yᵣ = view(Φ.G.y, J)
    #perform initial 4 interpolations
    zₓ = (
        cubic(x, xᵣ, @view Φ.G.Z[I,J[1]]),
        cubic(x, xᵣ, @view Φ.G.Z[I,J[2]]),
        cubic(x, xᵣ, @view Φ.G.Z[I,J[3]]),
        cubic(x, xᵣ, @view Φ.G.Z[I,J[4]])
    )
    #final interpolation
    cubic(y, yᵣ, zₓ)
end

Base.getindex(Φ::BicubicInterpolator, i, j) = Φ.G.Z[i,j]
function Base.setindex!(Φ::BicubicInterpolator, v, i, j)
    Φ.G.Z[i,j] = v
end
function Base.copy(Φ::BicubicInterpolator)
    BicubicInterpolator(Φ.G, Φ.boundaries, Ref(1), Ref(1))
end


# # [PDE Discovery : Heat equations](@id heat_equation)
#
# Similar to the discovery of ODE and DAE systems, sparse regression can be used to discover the underlying
# equations for partial differential equations as well as proposed by [PDEFind](https://www.science.org/doi/10.1126/sciadv.1602614). The following example shows how this
# can be achieved with DataDrivenDiffEq.jl, using the heat equation with Dirichlet boundary conditions
# with the analytical soultion $u(x,t) = sin(2\pi x) exp^{-(2\pi t)^2t}$.
# We start by defining the system and generate some data.

using DataDrivenDiffEq
using LinearAlgebra
using ModelingToolkit
using OrdinaryDiffEq
using DiffEqOperators
#md using Plots

u_analytic(x, t) = sin(2*π*x) * exp(-t*(2*π)^2)
nknots = 100
h = 1.0/(nknots+1)
knots = range(h, step=h, length=nknots)
ord_deriv = 2
ord_approx = 2

const bc = Dirichlet0BC(Float64)
const Δ = CenteredDifference(ord_deriv, ord_approx, h, nknots)

t0 = 0.0
t1 = 1.0
u0 = u_analytic.(knots, t0)

step(u,p,t) = Δ*bc*u
prob = ODEProblem(step, u0, (t0, t1))
alg = KenCarp4()
de_solution = solve(prob, alg);

#md plot(de_solution, legend = nothing)

# Using DiffEqOperators.jl, we can define the difference operators up to order $n =4$ and vectorize the result.

∂U = reduce(vcat, map(1:4) do n
    δ = CenteredDifference(n, ord_approx, h, nknots)
    reshape(
        δ*bc*Array(de_solution), 1, prod(size(de_solution))
        )
    end)

# Next we collect the discretized data samples, their time derivatives and define a DataDrivenProblem

U = reshape(Array(de_solution), 1, prod(size(de_solution)))
∂ₜU = reshape(Array(de_solution(de_solution.t, Val{1})), 1, prod(size(de_solution)))

problem = DataDrivenProblem(
    U, DX = ∂ₜU, U = ∂U
)

# We choose to model the spatial derivatives $\frac{d^n u}{dx^n}$ as exegenous signals (controls), which are directly substituted into the
# Basis defined in the following

@parameters t x
@variables u(x, t)
∂u = map(1:4) do n
    d = Differential(x)^n
    d(u)
end

basis = Basis([monomial_basis([u], 5);∂u], [u], independent_variable = t, controls = ∂u);
println(basis) #hide

# Afterwards, we define a sampler for the available data which performs a 80-20 train-test split and partions the training
# data into 10 batches and solve the sparse regression using STLSQ.

sampler = DataSampler(Split(ratio = 0.8), Batcher(n = 10))
solution = solve(problem, basis, STLSQ(1e-2:1e-2:5e-1), sampler = sampler, by = :best)

# As we can see, the heat equation is recovered correclty
result(solution)
println(result(solution)) #hide

#md # ## [Copy-Pasteable Code](@id cartpole_copy_paste)
#md #
#md # ```julia
#md # @__CODE__
#md # ```

## Test #src
for r_ in [solution] #src
    @test all(l2error(r_) .< 0.5) #src
    @test all(aic(r_) .> 1e3) #src
    @test all(determination(r_) .>= 0.9) #src
end #src


#!/usr/bin/env julia

"""
# Problem 6: Sum square differences

The sum of the squares of the first ten natural numbers is,

    1² + 2² + ... + 10² = 385

The square of the sum of the first ten natural numbers is,

    (1 + 2 + ... + 10)² = 55² = 3025

Hence the difference between the sum of the squares of the first ten natural
numbers and the square of the sum is 3025 − 385 = 2640.

Find the difference between the sum of the squares of the first `n` natural
numbers and the square of the sum.
"""
module Problem006

include("Sequences.jl")

"""
    solve(; n::Integer = 100)

Returns the solution for [`Problem006`](@ref) with the given parameters.
"""
function solve(; n::Integer = 100)
    return Sequences.triangular(n)^2 - Sequences.sum_of_squares(n)
end

end


# # Phase recovery using MaxCut
#
# In this example, we relax the phase retrieval problem similar to the classical
# [MaxCut](http://www-math.mit.edu/~goemans/PAPERS/maxcut-jacm.pdf) semidefinite
# program and recover the phase of the signal given the magnitude of the linear
# measurements.
#
# Phase recovery has wide applications such as  in X-ray and crystallography
# imaging, diffraction imaging or microscopy and audio signal processing. In all
# these applications, the detectors cannot measure the phase of the incoming wave
# and only record its amplitude i.e complex measurements of a signal
# $x \in \mathbb{C}^p$ are obtained from a linear injective operator $A$, **but we
# can only measure the magnitude vector $Ax$, not the phase of $Ax$**.
#
# Recovering the phase of $Ax$ from $|Ax|$ is a **nonconvex optimization problem**. Using results from [this paper](https://arxiv.org/abs/1206.0102), the problem can be relaxed to a (complex) semidefinite program (complex SDP).
#
# The original reprsentation of the problem is as follows:
#
# $$
# \begin{array}{ll}
#   \text{find} & x \in \mathbb{C}^p \\
#     \text{subject to} & |Ax| = b
# \end{array}
# $$
#
# where $A \in \mathbb{C}^{n \times p}$ and $b \in \mathbb{R}^n$.

# In this example, **the problem is to find the phase of $Ax$ given the value $|Ax|$**.
# Given a linear operator $A$ and a vector $b= |Ax|$ of measured amplitudes,
# in the noiseless case, we can write $Ax = \text{diag}(b)u$ where
# $u \in \mathbb{C}^n$  is a phase vector, satisfying
# $|\mathbb{u}_i| = 1$ for $i = 1,\ldots, n$.
#
# We relax this problem as Complex Semidefinite Programming.
#
# ### Relaxed Problem similar to [MaxCut](http://www-math.mit.edu/~goemans/PAPERS/maxcut-jacm.pdf)
#
# Define the positive semidefinite hermitian matrix
# $M = \text{diag}(b) (I - A A^*) \text{diag}(b)$. The problem is:
#
# $$
# \begin{array}{ll}
#   \text{minimize} & \langle U, M \rangle \\
#     \text{subject to} & \text{diag}(U) = 1\\
#     & U \succeq 0
# \end{array}
# $$
#
# Here the variable $U$ must be hermitian ($U \in \mathbb{H}_n $),
# and we have a solution to the phase recovery problem if $U = u u^*$
# has rank one. Otherwise, the leading singular vector of $U$ can be used
# to approximate the solution.

using Convex, SCS, LinearAlgebra

n = 20
p = 2
A = rand(n, p) + im * randn(n, p)
x = rand(p) + im * randn(p)
b = abs.(A * x) + rand(n)

M = diagm(b) * (I(n) - A * A') * diagm(b)
U = ComplexVariable(n, n)
objective = inner_product(U, M)
c1 = diag(U) == 1
c2 = U in :SDP
p = minimize(objective, c1, c2)
solve!(p, SCS.Optimizer; silent_solver = true)
evaluate(U)

#-

# Verify if the rank of $U$ is 1:
B, C = eigen(evaluate(U));
length([e for e in B if (abs(real(e)) > 1e-4)])

#-

# Decompose $U = uu^*$ where $u$ is the phase of $Ax$
u = C[:, 1];
for i in 1:n
    u[i] = u[i] / abs(u[i])
end
u


using Symbolics, SymbolicUtils

function solve_newton(f, x, x₀; abstol=1e-8, maxiter=50)
    xₙ = Float64(x₀)
    fₙ₊₁ = x - f / Symbolics.derivative(f, x)

    for i = 1:maxiter
        xₙ₊₁ = substitute(fₙ₊₁, Dict(x => xₙ))
        if abs(xₙ₊₁ - xₙ) < abstol
            return xₙ₊₁
        else
            xₙ = xₙ₊₁
        end
    end
    return xₙ₊₁
end

n = 2
@variables ϵ a[1:n]

x = 1 + a[1]*ϵ + a[2]*ϵ^2

  eq = x^5 + ϵ*x - 1

expand(eq)

function collect_powers(eq, x, ns; max_power=100)
    eq = substitute(expand(eq), Dict(x^j => 0 for j=last(ns)+1:max_power))

    eqs = []
    for i in ns
        powers = Dict(x^j => (i==j ? 1 : 0) for j=1:last(ns))
        push!(eqs, substitute(eq, powers))
    end
    eqs
end

eqs = collect_powers(eq, ϵ, 1:2)

substitute(expand(eq), Dict(
  ϵ => 0,
  ϵ^2 => 1,
  ϵ^3 => 0,
  ϵ^4 => 0,
  ϵ^5 => 0,
  ϵ^6 => 0,
  ϵ^7 => 0,
  ϵ^8 => 0)
)

function solve_coef(eqs, ps)
    vals = Dict()

    for i = 1:length(ps)
        eq = substitute(eqs[i], vals)
        vals[ps[i]] = Symbolics.solve_for(eq ~ 0, ps[i])
    end
    vals
end

solve_coef(eqs, a)

X = 𝜀 -> 1 + a[1]*𝜀 + a[2]*𝜀^2

def_taylor(x, ps) = sum([a*x^i for (i,a) in enumerate(ps)])
def_taylor(x, ps, p₀) = p₀ + def_taylor(x, ps)

n = 3
@variables ϵ M a[1:n]
x = def_taylor(ϵ, a, M)

expand_sin(x, n) = sum([(isodd(k) ? -1 : 1)*(-x)^(2k-1)/factorial(2k-1) for k=1:n])

expand_sin(0.1, 10) ≈ sin(0.1)

eq = x - ϵ * expand_sin(x, n) - M

eqs = collect_powers(eq, ϵ, 1:n)

vals = solve_coef(eqs, a)

x′ = substitute(x, vals)
X = (𝜀, 𝑀) -> substitute(x′, Dict(ϵ => 𝜀, M => 𝑀))
X(0.01671, π/2)


### A Pluto.jl notebook ###
# v0.14.2

using Markdown
using InteractiveUtils

# ╔═╡ 02d67371-b1e3-4ac9-b4d1-cdca1c85bd18
using Distributions, Statistics

# ╔═╡ 7c8394db-271e-4d0b-a3db-b3e3a659ec0a
using Gaston; set(showable="svg")

# ╔═╡ bf7ce886-e6ac-45e7-a0f5-98e36c8974c7
using LightGraphs, GraphPlot

# ╔═╡ 7a514f20-e7cc-43da-96f6-cc4f7244166b
using PlutoUI

# ╔═╡ c7a5ab0b-aba0-4b54-abe9-702b0bad4a8e
using SimpleWeightedGraphs

# ╔═╡ ea0dc819-e860-4863-b9ea-81d12d92c4ab
md"[Reference Chapter](https://www.deeplearningbook.org/contents/prob.html)"

# ╔═╡ e84a31da-8ce7-11eb-1b6a-0f1431d8c96e
md"# Probability"

# ╔═╡ 48727d46-8cf8-11eb-25a0-73c248dab2da
md"
The probability of an event A, given B

$ P(A|B) = \frac{P(A \cap B)}{P(B)}$

The probability of B, given A

$ P(B|A) = \frac{P(B \cap A)}{P(A)}$

We can rewrite the above $ 2 $ as --

$ \frac{P(A|B)}{P(B|A)} = \frac{P(A)}{P(B)} $
"

# ╔═╡ af0e8170-9163-11eb-2e48-4dcc7c853090
md"**What does $P(A \cap B)$ mean?**"

# ╔═╡ dde86ab8-9163-11eb-173a-0d0fe5d297ad
md"Probability of event A _and_ event B occurring."

# ╔═╡ 51cf0da0-8cf8-11eb-13d1-53bb6bd3892a
md"
#### Question --

In 5 coin tosses, whats the chance that we get exactly 3 heads, given that we've atleast one head?
"

# ╔═╡ 5c51aa26-8cf8-11eb-0add-a1b7f41216f6
md"
So... in how many ways can I rearrange 2 squares, in a collection of 3 objects, consisting of 2 squares and a circle?

⭘ ◻ ◻, ◻ ◻ ⭘, ◻ ⭘ ◻

3 ways, without repetition

Well... what about with repetition?

6 ways(if you consider the two squares to be 2 different objects)

So how do we generalize this?

$ ^{n}P_{r} = \frac{n!}{(n-r)!}$

where $ ^{n}P_{r} $ represents the permutation where we've to rearrange r objects, in a collection of n objects, and repetition is allowed.

$ ^{n}C_{r} = \frac{n!}{r!(n-r)!}$

where `` ^{n}C_{r} `` represents the permutation where we've to rearrange r objects, in a collection of n objects, and repetition is _not_ allowed.

"

# ╔═╡ 72edf972-8cf8-11eb-2441-75b6edc68854
md"For the above question... We've $ 2^5 $ or $ 32 $ possibilities after we toss the coin 5 times, or $ 32 $ possible combination of heads and tails! Since one of the possibilities is TTTTT, and we already know from the information we have, that we've at least one head, TTTTT is not possible, so we've a total of $31$ possibilities.

Out of these $31$ possibilities, we've to find the ones with exactly three heads. Going with the above formula, we have $^{5}C_{3}$ possibilities of getting exactly three heads in five tosses, which is equal to
"

# ╔═╡ c26eb560-9167-11eb-02d3-8b4212f378e6
binomial(5,3)

# ╔═╡ 6d6ead0a-9170-11eb-049b-2dfaf08ba40e
md"So the probability of getting exactly 3 heads is $10/31$"

# ╔═╡ 5e59cfb6-9170-11eb-0215-2390218aac63
md"The below code finds all possible results we can obtain, after flipping a coin 5 times, and stores it in variable `a`. It then removes the possibility that the result is TTTTT, as per given in the question, from `a`."

# ╔═╡ 6fd98eb6-916b-11eb-19a4-5368d9cc2b31
begin a = [prod.(collect(Iterators.product(['H', 'T'], ['H', 'T'], ['H', 'T'], ['H', 'T'], ['H', 'T'])))[i] for i in 1:2^5]
	deleteat!(a,findall(a .== "TTTTT"))
end

# ╔═╡ 980bb77e-9170-11eb-3d61-19044344aa37
md"The below code finds the number of results where we get heads exactly 3 times, and stores the outcomes in the variable `b`"

# ╔═╡ 0c0ac120-916b-11eb-1029-cb8615bca99c
b = a[[(count("H", a[i]) == 3) for i in 1:length(a)]]

# ╔═╡ 4ef0b236-9172-11eb-0813-e12589886a31
md"Counting the number of elements in `a` and `b`"

# ╔═╡ 47f31ab4-9172-11eb-1bc4-f916418a47db
length(b), length(a)

# ╔═╡ eced0ac6-9171-11eb-0f2a-cd5fa1bd818d
md"We get our answer equal to $ $(length(b)) / $(length(a)) $"

# ╔═╡ 3f168c24-9173-11eb-1cf8-7338a422ff69
md"## Random Variable"

# ╔═╡ f6b2bf60-9173-11eb-145d-2f1f47b7eb30
md"""
!!! question "Consider this example"
	I've a blue bag, and a red bag.

	The blue bag has 3 green balls and 5 yellow balls, and the red bag has 6 green balls and 4 yellow balls.

	What is the probability that when I choose a random bag and draw a ball, its a green ball?
"""

# ╔═╡ 79557c7c-9173-11eb-3c53-4fa9ad783e8a
md"""
In the above example, the color of the bag and the color of balls are _random variables_. Let's call the color of bag, B, and the color of balls, A.

We've to find out $P(A=🟢)$.

The probability that I choose a blue bag is, denoted by $P(B=$"blue"$)$, and the probability that I choose the red bag is denoted by $P(B=$"red"$)$.

Mind that the random variable _color of bag_ can have two outcomes. It can either be blue or yellow with equally likely chances with the total probability bound to be $1$. Which means, $P(B=$"blue"$)$ $= P(B=$"red"$)$ and $P(B=$"blue"$) + P(B=$"red"$) = 1$.
Which means $P(B=$"blue"$) = P(B=$"red"$) = 0.5$.

Try thinking of it this way, when an unbiased coin is tossed. Very similar to the above case, you've two expected outcomes, C = "Heads" or C = "Tails", where C is a random variable denoting the side when the coin is tossed. Here too, $P(C = $"Heads" $) = P(C = $"Tails" $) = 0.5$.

"""

# ╔═╡ 35cfa732-9238-11eb-3f47-0d921b5d6eeb
md"## Types of random variables and Joint Probability"

# ╔═╡ b4374be0-9173-11eb-09d2-cbab6657cdaf
md"""
Returning to our previous example,

$ P(A=🟢) = P(B= $ "red"$, A=🟢) + P(B=$"blue"$, A=🟢)$

Probability that the ball we chose is a green ball = (Probability that we chose the red bag, and a green ball) + (Probability that we chose the blue bag, and a green ball)

$ P(A=🟢) = \frac{1}{2} \times \frac{3}{8}  + \frac{1}{2} \times \frac{6}{10}$

$ P(A=🟢) = \frac{39}{80}$ or $ $(39/80) $

"""

# ╔═╡ 70e11290-923b-11eb-2365-f58a04d0f844
md"""
The probability of two or more events occuring at the same time, is called **Joint Probability**

You see it in the above example and you see it in the below question.
"""

# ╔═╡ 758a1460-8cf8-11eb-0ab7-7dfcfac203f9
md"""
!!! question "An example"
	In a particular hospital during Flu season, it was observed that 10% of the patients had Cold. Among the patients with cold, 80% had Flu and among the rest, only 5% had flu. Given that a particular patient had Flu, what is the probability that he had Cold?
"""

# ╔═╡ 806857e8-8cf8-11eb-3756-bff672168730
md"""
Probability that a patient had cold = $ 0.1 $

Probability that a patient had cold, and had flu = $ 0.1 × 0.8 $ = $ 0.08 $

Probability that a patient didn't have cold, and had flu = $ 0.9 × 0.05 $ = $ $(round((0.9*0.05); digits=3)) $

Probability that a patient has cold, given that he had flu = $\frac{Probability\;that\;a\;patient\;has\;cold\;and\;flu}{Probability\;that\;a\;patient\;has\;flu} = \frac{0.08}{0.08+0.045}$ = $ \frac{16}{25} $
"""

# ╔═╡ 7bc55826-92de-11eb-258b-31d565464c56
md"""
!!! question "Question"
	A man arrives at a bus stop at a random time (that is, with no regard for the scheduled service) to catch the next bus. Buses run every $30$ minutes without fail, hence the next bus will come any time during the next $30$ minutes with evenly distributed probability (a uniform distribution). Find the probability that a bus will come within the next $10$ minutes.
"""

# ╔═╡ a2128026-92de-11eb-102f-d924f0ceee27
md"""
In the above question, the bus can arrive any time within the next 30minutes. It has equal chances of coming at the $0.1$min or $0.01$min or $25.005$min. Every value we choose for the arrival of the bus has equal proability. For example, the probability of the bus arriving at $1$min is the same as the bus arriving at $29.909$min. Since there're infinite possibilities, for a particular given time, the probability always approaches $0$. Here, if we assume the random variable to be _the time at which the bus arrives_, its number of possible outcomes can't be counted. Such random variables are called **continous random variables**. Similarly, the random variables we encountered before this had countable number of outcomes, and were called **discrete random variables**. $𝑝(x)$ is also called the **probability density function**, and $P(x)$ is also known as the **probability mass function**.

When it comes to **continous random variables**, we try finding the probability of the outcome falling in a particular interval, rather than a particular value as in the case of **discrete random variables**.
"""

# ╔═╡ 749c569c-92e0-11eb-14c5-8301f513cdad
md"""
In this question, since all outputs are equally likely, if we divide the time interval in three parts, the outcome would have equal chances of falling in any of the three intervals. That means, the time at which the bus arrives can be within $0$ to $10$ minutes, $10$ to $20$ minutes, or $20$ to $30$ minutes, with equally likely chances. And since the total probability is always one, following the notation from the book,

$𝑝(0<X<10) = 𝑝(10<X<20) = 𝑝(20<X<30)$

And since the total probability is $1$, $𝑝(0<X<10) = \frac{1}{3}$
"""

# ╔═╡ e0c4991e-92e1-11eb-1cc3-31c0fae42abf
md"Similarly, if we've to choose a number from real numbers between $1$ to $10$, the random variable denoted by X, would be a **continous random variable**.

Suppose, we've to find out, what's the probaility that X is less than 3?

That would be $𝑝(X<3) = 𝑝(1<X<3) = \frac{2}{9}$. How?

Well, if we try dividing the intervals at every integer, we've

$𝑝(1<X<2) = 𝑝(2<X<3) = 𝑝(3<X<4) = 𝑝(4<X<5) = 𝑝(5<X<6) = 𝑝(6<X<7) = 𝑝(7<X<8) = 𝑝(8<X<9) = 𝑝(9<X<10) = \frac{1}{9}$

and $𝑝(X<3) = 𝑝(1<X<3) = 𝑝(1<X<2) + 𝑝(2<X<3)$."

# ╔═╡ 1c95027c-930e-11eb-2a05-8322da402360
a1 = rand(0:0.00001:30, 1000000)

# ╔═╡ a6073126-930c-11eb-2785-79a71976e4d5
b1 = a1[a1 .<= 10]

# ╔═╡ f9c89c3c-930c-11eb-278d-9bf0ee036c67
length(b1)/length(a1)

# ╔═╡ 16b2a794-9311-11eb-3604-f1efc56fe690
md"Which is pretty close to our answer, $\frac{1}{3}$"

# ╔═╡ 332020be-9311-11eb-16c5-03374a5c7112
a2 = rand(1:0.00001:10, 10000)

# ╔═╡ 420afebe-9311-11eb-1cd2-2b0ddbb58716
b2 = a2[a2 .<= 3]

# ╔═╡ 5e578c2c-9311-11eb-22a0-2f57b00d7958
"$(length(b2)/length(a2)) which is close to $(2/9)"

# ╔═╡ 55eee81a-f8cc-4503-adce-3153342baf00
md"### Let's set up some terms"

# ╔═╡ 6f8a52ba-f673-47e0-890a-83a7b2e338d9
md"""
(Ⅰ) **Probability Distribution**

_"In probability theory and statistics, a probability distribution is the mathematical function that gives the probabilities of occurrence of different possible outcomes for an experiment."_

$ \,\,\,\,\,\,\,\,\,\,\,\,\,\,\, $ --[Wikipedia](https://en.wikipedia.org/wiki/Probability_distribution)
"""

# ╔═╡ 48bffce6-6ba1-44b8-bd70-ff11b708a23d
md"""
(Ⅱ) **Mean**

Its the average of a probability distribution of a discrete random variable.
"""

# ╔═╡ b5b45043-cbea-4848-862c-5b39bf6fd2bb
md"""
(Ⅲ) **Standard Deviation**

_"In statistics, the standard deviation is a measure of the amount of variation or dispersion of a set of values. A low standard deviation indicates that the values tend to be close to the mean of the set, while a high standard deviation indicates that the values are spread out over a wider range."_

$ \,\,\,\,\,\,\,\,\,\,\,\,\,\,\, $ --[Wikipedia](https://en.wikipedia.org/wiki/Standard_deviation)

"""

# ╔═╡ a75111b9-063a-4158-a78f-7f1107db028c
md"""
(Ⅳ) **Expectation**

The expectation $E$ is defined for a discrete random variable as

$E = ∑P(x)f(x)$
where it tells the expected value of some function $f(x)$, if its probability distribution is $P(x)$.

Similarly, the expectation E is defined for a discrete random variable as

$E = ∫p(x)f(x)dx$
where it tells the expected value of some function $f(x)$, if its probability distribution is $p(x)$.
"""

# ╔═╡ 1683d427-5433-4d85-b65e-4b86cc61af14
md"""
(Ⅴ) **Variance**

**Definition from the book:** _"The variance gives a measure of how much the values of a function of a random
variable x vary as we sample diﬀerent values of x from its probability distribution."_

$Var(f(x)) = E[(f(x) − E[f(x)])^2]$
**_Note_**:
$Var(f(x)) = (Standard\,Deviation)^2$
"""

# ╔═╡ 3a2c5692-ee39-4f26-b094-a6094dd88b68
md"""
### Some commonly used Probability Distributions
"""

# ╔═╡ f5af9c66-ee47-4384-a68a-60711f966557
md"#### [Bernoulli Distribution](https://juliastats.org/Distributions.jl/stable/univariate/#Distributions.Bernoulli)"

# ╔═╡ 203d14b2-d7b6-4ac4-b1f8-421a2e80c51a
bern = Bernoulli()

# ╔═╡ e18cdf58-13a9-4f98-ab97-6c5cdcc3ee7c
mean(bern), var(bern) #mean and variance of bernoulli distribution

# ╔═╡ a0dfaf17-1cbf-45e1-be14-8302f68b2b6a
md"#### [Multinomial Distribution](https://juliastats.org/Distributions.jl/stable/multivariate/#Distributions.Multinomial)"

# ╔═╡ 052c45aa-1735-4cba-b911-28c81d7e2850
let
	m = Multinomial(5, 12) #setting up the mulinomial distribution, with n=5
	m, mean(m)
end

# ╔═╡ f2a98e2b-328c-4a8d-bfeb-e5dd7cbb3014
let
	m = Multinomial(12, [0.3, 0.2, 0.45, 0.05])
	m, var(m)
end

# ╔═╡ 6f8963c0-8c36-43ab-9a6d-e814aaba2d81
md"#### Multinoulli Distribution"

# ╔═╡ 4ee83e69-db95-4724-b963-de7dd2445b02
multinoulli = Multinomial(1, 6) #setting up the mulinoulli distribution with k=6

# ╔═╡ 1de8aded-81e9-4f87-baa2-c571f31cb31b
md"""
#### Gaussian Distribution
"""

# ╔═╡ d038c84a-af8b-4da7-974d-0c023e40fcfb
md"[Univariate Normal Distribution](https://juliastats.org/Distributions.jl/stable/univariate/#Distributions.Normal)"

# ╔═╡ d5b23092-a9fe-44c3-ae9d-bd7186cbdf07
uninorma = Normal(1, 0.2)

# ╔═╡ f7f637b6-2f6e-47dc-b1ca-baf1669f6681
mean(uninorma), var(uninorma)

# ╔═╡ 7cff52e3-5063-47a6-8d0e-03c333ee401b
md"[Multivariate Normal Distribution](https://juliastats.org/Distributions.jl/stable/multivariate/#Distributions.AbstractMvNormal)"

# ╔═╡ 0235d738-c7e4-48b0-b7fd-9b6d31a2bbbc
multinorma = MvNormal(3, 0.8)

# ╔═╡ 0b67edef-d59a-44ed-9226-3c857e78b0e7
mean(multinorma), var(multinorma)

# ╔═╡ d3faf6d4-6584-476f-bb62-d34ccb701bc4
rand(bern), rand(multinorma), rand(uninorma), rand(multinoulli), rand(Multinomial(5, 12))

# ╔═╡ ce6a1226-923c-11eb-074b-6d708b9773ff
md"""
!!! question "Question"
	Heights of  $25$-year-old men in a certain region have mean $69.75$ inches and standard deviation  $2.59$  inches. These heights are approximately normally distributed. Thus the height  X  of a randomly selected  $25$-year-old man is a normal random variable with mean $μ=69.75$ and standard deviation  $σ=2.59$. Find the probability that a randomly selected $25$-year-old man is more than $69.75$ inches tall.
"""

# ╔═╡ 6489f711-b127-4283-8171-9478d4c48a0c
n = Normal(69.75, 2.59)

# ╔═╡ fc4769e0-f6c1-473c-b675-13ec737a5572
inputrange = quantile(n, 0.0001):0.1:quantile(n, 1-0.0001) #outputs a set of values between the minimum and maximum of an infinite distribution, with an interval of 0.1 in this case

# ╔═╡ 5a8346f0-ea6f-43e6-916b-041ee5220367
 output = pdf(n, inputrange)

# ╔═╡ e1be7ec4-6121-401b-b338-37cd34591586
let
	plot(inputrange, output, Axes(xtics=60.75:3:80, title="'Density function of the height of a 25 year old man'"), w=:l)
	d = findfirst(output .== maximum(output))
	plot!(repeat([inputrange[d]],2), [0, output[d]], w=:lp, pt=7, lt=:dash)
end

# ╔═╡ 251004a8-fb13-4e92-8c08-27286351753f
md"""
As you can see, the plot is symmetric with respect to $x=69.75$. It means, the probability of a randomly selected $25$ year old person being greater than $69.75$ inches in height is same as the probability of that person's height being less than $69.75$ inches.

Hence, outcomes are equally likely and we can say that

$p(x>69.75) = p(x<69.75) = 0.5$
"""

# ╔═╡ 291d53fc-93ec-4926-80ab-a276b025ef82
md"""
## Likelihood
"""

# ╔═╡ fd6e90a0-36f9-4055-a69b-5809cd05b837

# ╔═╡ 3d141deb-bf7c-4ef3-88db-1b7e31ba6033
md"""
#### Try Yourself
1. Try making the Dirac Delta function in Julia.
2. Explore [this](https://stats.libretexts.org/Bookshelves/Introductory_Statistics/Book%3A_Introductory_Statistics_(Shafer_and_Zhang)/05%3A_Continuous_Random_Variables/5.01%3A_Continuous_Random_Variables) website, look at solved and unsolved questions, try solving them in julia.
3. Check out the [Exponential Distribution](https://juliastats.org/Distributions.jl/latest/univariate/#Distributions.Exponential) and [Laplace Distribution](https://juliastats.org/Distributions.jl/latest/univariate/#Distributions.Laplace) from Distributions.jl
4. Explore [Gaston](https://mbaz.github.io/Gaston.jl/stable/) more, as it'll be used throughout the tutorial. Try getting familiar with it.
"""

# ╔═╡ d4bf1c39-6281-4ddf-b87e-bf4e3cefea3e
md"# Information Theory"

# ╔═╡ 747b4b5b-98e7-4171-bfb0-b5acc792eda3
md"""
The self-information of an event $X=x$ denoted by 𝐼, is given by
"""

# ╔═╡ 75b07946-f493-49f0-928c-afee87fb22ad
I(x) = -log(x)

# ╔═╡ 4cf8dd4f-af3b-421f-ab11-e3036d60cc29
md"where x denotes the probability of a random variable's outcome being $x$."

# ╔═╡ 2454387d-df18-4932-ad48-691e5583a21e
md"with the Shanon Entropy given by"

# ╔═╡ 50920106-7a39-4563-a1c0-cdd5d43ebb37
H(x) = mean(I(x))

# ╔═╡ e8140a83-904a-46b6-a8e8-8a7151b85c43
md"Its mostly theoritical stuff. In this section, I'd like to introduce the reader to Graphs"

# ╔═╡ 76dc38a7-d06e-41c4-8da5-b15e3ebe54de
md"""
## Graphs
_In mathematics, and more specifically in graph theory, a graph is a structure amounting to a set of objects in which some pairs of the objects are in some sense "related". The objects correspond to mathematical abstractions called **vertices** (also called **nodes** or **points**) and each of the related pairs of vertices is called an **edge** (also called link or line). Typically, a graph is depicted in diagrammatic form as a set of dots or circles for the vertices, joined by lines or curves for the edges._

_The edges may be **directed** or **undirected**. For example, if the vertices represent people at a party, and there is an edge between two people if they shake hands, then this graph is undirected because any person A can shake hands with a person B only if B also shakes hands with A. In contrast, if any edge from a person A to a person B corresponds to A owes money to B, then this graph is **directed**, because owing money is not necessarily reciprocated. The former type of graph is called an **undirected** graph while the latter type of graph is called a **directed** graph._

$ \,\,\,\,\,\,\,\,\, $ --[Wikipedia](https://en.wikipedia.org/wiki/Graph_(discrete_mathematics))
"""

# ╔═╡ 79f9cf36-1f2c-456f-a71c-d0bac25a5aad
md"For an example, lets take a look ath the below graph."

# ╔═╡ 87f96fa0-c27e-45a6-8452-f67f952b737d
Resource("https://upload.wikimedia.org/wikipedia/commons/thumb/b/bf/Undirected.svg/220px-Undirected.svg.png")

# ╔═╡ 1d009804-1858-4109-a75b-16ae5523bfa2
md"The three blue circles are **nodes**. While the lines connecting them are **edges**"

# ╔═╡ cdf1583c-6964-4064-9434-8be1d4c397d9
md"Its an **undirected graph**, since the edges have no orientation."

# ╔═╡ 68d60332-1aa2-4668-bfbe-fefa588fe582
md"Lets try making one in LightGraphs"

# ╔═╡ b7c3798f-5fbb-4a87-8299-28e4c4c28fec
let
	g = SimpleGraph(3) #Makes an undirected graph
	add_edge!(g, 1, 2); #Adding an edge from 1 to 2
	add_edge!(g, 2, 3);
	add_edge!(g, 1, 3);
	gplot(g)
end

# ╔═╡ cf8483c1-d805-4e99-b2f4-7dce289fa587
q = [[1,1,1], [1,1,1]]

# ╔═╡ e07b27ab-b1c5-41c6-b932-641bcd893324
append!(q[1], 1)

# ╔═╡ 236e38f8-acea-4d03-a34d-d502e6e8e32d
md"""
A direct graph is one where the edges have an orientation. From the above example, assume A owes money to B and B owes money to C.
"""

# ╔═╡ a686295f-6cf1-4da4-be0e-b1935892fc44
let
	g = SimpleDiGraph(3) #Makes an undirected graph
	add_edge!(g, 1, 2); #Adding an edge from 1 to 2
	add_edge!(g, 2, 3);
	gplot(g, nodelabel=["A", "B", "C"])
end

# ╔═╡ 26506fba-3d46-49e5-822b-725741003e17
md"
A **Weighted Graph** is a graph where weights are assigned to each edge.
Assume, in the above example, A owes `` \$54 `` to B and B owes `` \$89 `` to C.
"

# ╔═╡ df7e83b8-eb8b-411c-be35-07d5770fe9b1
let
	sources = [1,2]
	destinations = [2,3]
	weights = [54, 89]
	nodelabel=["A", "B", "C"]
	g = SimpleWeightedDiGraph(sources, destinations, weights) #Makes an undirected graph
	gplot(g, nodelabel=["A", "B", "C"], edgelabel=["５𝟦", "𝟾𝟫"])
end

# ╔═╡ 0119dc9f-8ccf-4776-a7e9-5f61b063d509
md"
### Try yourself
"

# ╔═╡ 0db8895b-cb4d-4a3d-9bae-1fd0f460fafe
md"""
1. Look for different types of graphs on the [wikipedia page](https://en.wikipedia.org/wiki/Graph_(discrete_mathematics)). Try making and visualizing them using [LightGraphs](https://github.com/JuliaGraphs/LightGraphs.jl)+[GraphPlot](https://github.com/JuliaGraphs/GraphPlot.jl).
2. Try visualizing both the structured probablistic models examples.
"""

# ╔═╡ Cell order:
# ╟─ea0dc819-e860-4863-b9ea-81d12d92c4ab
# ╟─e84a31da-8ce7-11eb-1b6a-0f1431d8c96e
# ╟─48727d46-8cf8-11eb-25a0-73c248dab2da
# ╟─af0e8170-9163-11eb-2e48-4dcc7c853090
# ╟─dde86ab8-9163-11eb-173a-0d0fe5d297ad
# ╟─51cf0da0-8cf8-11eb-13d1-53bb6bd3892a
# ╟─5c51aa26-8cf8-11eb-0add-a1b7f41216f6
# ╟─72edf972-8cf8-11eb-2441-75b6edc68854
# ╠═c26eb560-9167-11eb-02d3-8b4212f378e6
# ╟─6d6ead0a-9170-11eb-049b-2dfaf08ba40e
# ╟─5e59cfb6-9170-11eb-0215-2390218aac63
# ╠═6fd98eb6-916b-11eb-19a4-5368d9cc2b31
# ╟─980bb77e-9170-11eb-3d61-19044344aa37
# ╠═0c0ac120-916b-11eb-1029-cb8615bca99c
# ╟─4ef0b236-9172-11eb-0813-e12589886a31
# ╠═47f31ab4-9172-11eb-1bc4-f916418a47db
# ╟─eced0ac6-9171-11eb-0f2a-cd5fa1bd818d
# ╟─3f168c24-9173-11eb-1cf8-7338a422ff69
# ╟─f6b2bf60-9173-11eb-145d-2f1f47b7eb30
# ╟─79557c7c-9173-11eb-3c53-4fa9ad783e8a
# ╟─35cfa732-9238-11eb-3f47-0d921b5d6eeb
# ╟─b4374be0-9173-11eb-09d2-cbab6657cdaf
# ╟─70e11290-923b-11eb-2365-f58a04d0f844
# ╟─758a1460-8cf8-11eb-0ab7-7dfcfac203f9
# ╟─806857e8-8cf8-11eb-3756-bff672168730
# ╟─7bc55826-92de-11eb-258b-31d565464c56
# ╟─a2128026-92de-11eb-102f-d924f0ceee27
# ╟─749c569c-92e0-11eb-14c5-8301f513cdad
# ╟─e0c4991e-92e1-11eb-1cc3-31c0fae42abf
# ╠═1c95027c-930e-11eb-2a05-8322da402360
# ╠═a6073126-930c-11eb-2785-79a71976e4d5
# ╠═f9c89c3c-930c-11eb-278d-9bf0ee036c67
# ╟─16b2a794-9311-11eb-3604-f1efc56fe690
# ╠═332020be-9311-11eb-16c5-03374a5c7112
# ╠═420afebe-9311-11eb-1cd2-2b0ddbb58716
# ╠═5e578c2c-9311-11eb-22a0-2f57b00d7958
# ╟─55eee81a-f8cc-4503-adce-3153342baf00
# ╟─6f8a52ba-f673-47e0-890a-83a7b2e338d9
# ╟─48bffce6-6ba1-44b8-bd70-ff11b708a23d
# ╟─b5b45043-cbea-4848-862c-5b39bf6fd2bb
# ╟─a75111b9-063a-4158-a78f-7f1107db028c
# ╟─1683d427-5433-4d85-b65e-4b86cc61af14
# ╟─3a2c5692-ee39-4f26-b094-a6094dd88b68
# ╠═02d67371-b1e3-4ac9-b4d1-cdca1c85bd18
# ╟─f5af9c66-ee47-4384-a68a-60711f966557
# ╠═203d14b2-d7b6-4ac4-b1f8-421a2e80c51a
# ╠═e18cdf58-13a9-4f98-ab97-6c5cdcc3ee7c
# ╟─a0dfaf17-1cbf-45e1-be14-8302f68b2b6a
# ╠═052c45aa-1735-4cba-b911-28c81d7e2850
# ╠═f2a98e2b-328c-4a8d-bfeb-e5dd7cbb3014
# ╟─6f8963c0-8c36-43ab-9a6d-e814aaba2d81
# ╠═4ee83e69-db95-4724-b963-de7dd2445b02
# ╟─1de8aded-81e9-4f87-baa2-c571f31cb31b
# ╟─d038c84a-af8b-4da7-974d-0c023e40fcfb
# ╠═d5b23092-a9fe-44c3-ae9d-bd7186cbdf07
# ╠═f7f637b6-2f6e-47dc-b1ca-baf1669f6681
# ╟─7cff52e3-5063-47a6-8d0e-03c333ee401b
# ╠═0235d738-c7e4-48b0-b7fd-9b6d31a2bbbc
# ╠═0b67edef-d59a-44ed-9226-3c857e78b0e7
# ╠═d3faf6d4-6584-476f-bb62-d34ccb701bc4
# ╟─ce6a1226-923c-11eb-074b-6d708b9773ff
# ╠═7c8394db-271e-4d0b-a3db-b3e3a659ec0a
# ╠═6489f711-b127-4283-8171-9478d4c48a0c
# ╠═fc4769e0-f6c1-473c-b675-13ec737a5572
# ╠═5a8346f0-ea6f-43e6-916b-041ee5220367
# ╠═e1be7ec4-6121-401b-b338-37cd34591586
# ╟─251004a8-fb13-4e92-8c08-27286351753f
# ╟─291d53fc-93ec-4926-80ab-a276b025ef82
# ╠═fd6e90a0-36f9-4055-a69b-5809cd05b837
# ╟─3d141deb-bf7c-4ef3-88db-1b7e31ba6033
# ╟─d4bf1c39-6281-4ddf-b87e-bf4e3cefea3e
# ╟─747b4b5b-98e7-4171-bfb0-b5acc792eda3
# ╠═75b07946-f493-49f0-928c-afee87fb22ad
# ╟─4cf8dd4f-af3b-421f-ab11-e3036d60cc29
# ╟─2454387d-df18-4932-ad48-691e5583a21e
# ╠═50920106-7a39-4563-a1c0-cdd5d43ebb37
# ╟─e8140a83-904a-46b6-a8e8-8a7151b85c43
# ╠═bf7ce886-e6ac-45e7-a0f5-98e36c8974c7
# ╟─76dc38a7-d06e-41c4-8da5-b15e3ebe54de
# ╠═7a514f20-e7cc-43da-96f6-cc4f7244166b
# ╟─79f9cf36-1f2c-456f-a71c-d0bac25a5aad
# ╠═87f96fa0-c27e-45a6-8452-f67f952b737d
# ╟─1d009804-1858-4109-a75b-16ae5523bfa2
# ╟─cdf1583c-6964-4064-9434-8be1d4c397d9
# ╟─68d60332-1aa2-4668-bfbe-fefa588fe582
# ╠═b7c3798f-5fbb-4a87-8299-28e4c4c28fec
# ╠═cf8483c1-d805-4e99-b2f4-7dce289fa587
# ╠═e07b27ab-b1c5-41c6-b932-641bcd893324
# ╟─236e38f8-acea-4d03-a34d-d502e6e8e32d
# ╠═a686295f-6cf1-4da4-be0e-b1935892fc44
# ╟─26506fba-3d46-49e5-822b-725741003e17
# ╠═c7a5ab0b-aba0-4b54-abe9-702b0bad4a8e
# ╠═df7e83b8-eb8b-411c-be35-07d5770fe9b1
# ╟─0119dc9f-8ccf-4776-a7e9-5f61b063d509
# ╟─0db8895b-cb4d-4a3d-9bae-1fd0f460fafe

PlutoNotebooks/Probability and Information Theory.jl


export Eulerian
"""
`Eulerian(n,k)` returns the number of permutations of `{1,2,...,n}`
with `k` ascents.
"""
function Eulerian(n::Integer, k::Integer)::BigInt
    @assert (n>=0 && k>=0) "$n,$k must both be nonnegative"

    if n==0
        return big(0)
    end

    if k>n || k==0
        return big(0)
    end

    if n < 2
        return big(1)
    end

    if k==n      # includes the case n=k=0
        return big(1)
    end

    if k==1
        return big(1)
    end

    if _has(Eulerian,(n,k))
        return _get(Eulerian,(n,k))
    end

    val = (n-k+1)*Eulerian(n-1,k-1) + k*Eulerian(n-1,k)
    _save(Eulerian,(n,k),val)
    return val
end

_make(Eulerian,Tuple{Integer,Integer})


#unit step function
function u(t::Float64)::Float64 #unit step function
    return ifelse( t < 0, zero(t), one(t) )
end

#unnormalized Gaussian function
function 𝒩ᵤ(x::Float64; μ::Float64, σ::Float64)::Float64
    return exp(-0.5 * ((x - μ) / σ)^2)
end

#normalized Gaussian function
function 𝒩(x::Float64; μ::Float64, σ::Float64)::Float64
    return 1 / (σ * sqrt(2 * π)) * exp(-0.5 * ((x - μ) / σ)^2)
end

#Dirac delta (Gaussian approximation) amplitude normalized
function δn(t::Float64,σ::Float64=1.0e-11)::Float64
   return 𝒩ᵤ(t, μ=0.0, σ=σ)
end

#Dirac delta (Gaussian approximation) unit area
function δ(t::Float64,σ::Float64=1.0e-10)::Float64
   return 𝒩(t, μ=0.0, σ=σ)
end


"""
    abs_val(num)

Program to find the absolute value of a number

# Example
```julia
abs_val(-100) # returns 100
abs_val(0) # returns 0
abs(123.1) # returns 123.1
-1000 == abs_val(-1000) #returns false
1000 == abs_val(1000) #returns true
```

Contributed By:- [Ashwani Rathee](https://github.com/ashwani-rathee) and [Rratic](https://github.com/Rratic)
"""
function abs_val(num::N)where N<:Real
    return num < 0 ? -num : num
end

"""
    abs_max(x)

Program to find the max absolute value in a vector

# Example
```julia
abs_max([1,3,4]) # returns 4
abs_max([-3,1,2]) # returns -3
abs_max([-7,-3,6]) #returns -7
```

Contributed By:- [Ashwani Rathee](https://github.com/ashwani-rathee) and [Rratic](https://github.com/Rratic)
"""
function abs_max(x::Vector{N})where N<:Real
    _max=x[1]
    for i in x
        if abs(i)>abs(_max) _max=i end
    end
    return _max
end

"""
    abs_min(num)

Program to find the min absolute value in a vector

# Example
```julia
abs_min([1,3,4]) # returns 1
abs_min([-3,1,2]) # returns 1
abs_min([-7,-3,6]) #returns -3
```

Contributed By:- [Ashwani Rathee](https://github.com/ashwani-rathee) and [Rratic](https://github.com/Rratic)
"""
function abs_min(x::Vector{N})where N<:Real
    _min=x[1]
    for i in x
        if abs(i)<abs(_min) _min=i end
    end
    return _min
end


# # Constrained optimization

#=
In this example, we show how to differentiate through the solution of the following constrained optimization problem:
```math
\hat{y}(x) = \min_{y \in \mathcal{C}} f(x, y)
```
where ``\mathcal{C}`` is a closed convex set.
The optimal solution can be found as the fixed point of the projected gradient algorithm for any step size ``\eta``. This insight yields the following optimality conditions:
```math
F(x, \hat{y}(x)) = 0 \quad \text{with} \quad F(x,y) = \mathrm{proj}_{\mathcal{C}}(y - \eta \nabla_2 f(x, y)) - y
```
=#

using ImplicitDifferentiation
using Ipopt
using JuMP
using Zygote

using ChainRulesTestUtils  #src
using Test  #src

# ## Projecting onto the simplex

#=
We focus on minimizing ``f(x,y) = \lVert x - y \rVert_2^2``.
We also assume that ``\mathcal{C} = \Delta^n`` is the ``n``-dimensional probability simplex, because we know an exact procedure to compute the projection *and* its Jacobian.

Both of these procedures are outlined in [_From Softmax to Sparsemax: A Sparse Model of Attention and Multi-Label Classification_](https://arxiv.org/abs/1602.02068).
You can find an implementation in the function [`simplex_projection`](@ref).
Because this function involves a call to `sort`, standard AD backends cannot differentiate through it, which is why we also had to define a chain rule for it.
=#

# ## Implicit function wrapper

# We now wrap a black box optimizer inside an `ImplicitFunction` to compare its implicit differentiation with the explicit procedure given above.

function forward(x)
    n = length(x)
    optimizer = optimizer_with_attributes(Ipopt.Optimizer, "print_level" => 0)
    model = Model(optimizer)
    @variable(model, y[1:n] >= 0)
    @constraint(model, sum(y) == 1)
    @objective(model, Min, sum((y .- x) .^ 2))
    optimize!(model)
    return value.(y)
end;

conditions(x, y) = simplex_projection(y - 0.1 * 2(y - x)) - y;

implicit = ImplicitFunction(forward, conditions);

# ## Testing

# Let us study the behavior of our implicit function.

x = rand(5)

# We can see that the forward pass computes the projection correctly, at least up to numerical precision.

hcat(simplex_projection(x), implicit(x))

# And the same goes for the Jacobian.

cat(Zygote.jacobian(simplex_projection, x)[1], Zygote.jacobian(implicit, x)[1]; dims=3)

# The following tests are not included in the docs.  #src

@test implicit(x) ≈ simplex_projection(x) atol = 1e-5  #src

@testset verbose = true "ChainRules" begin  #src
    test_rrule(simplex_projection, x; check_inferred=false)  #src
    test_rrule(implicit, x; atol=1e-2, check_inferred=false)  #src
end  #src


"""
Julia program to implement Maximum Sum Decreasing Subsequence
In this problem, given an array we have to find the maximum sum an decreasing subsequence of that array can make.
This problem is a slight modification to the Longest Decreasing subsequence problem.
The problem can be solved using Dynamic Programming
"""

function maximum_sum_decreasing_subsequence(arr, n)
    max_sum = 0
    dp = Int[]

    # Initialize the dp array with the array values, as the maximum sum
    # at each point is atleast as the value at that point
    for i in 1:n
        push!(dp, arr[i])
    end

    for i in 1:n
        for j in 1:(i-1)
            if(arr[i] < arr[j] && dp[i] < dp[j] + arr[i])
                dp[i] = dp[j] + arr[i]
            end
        end
    end

    # Now Find the maximum element in the dp array
    max_sum = findmax(dp)[1]
    return max_sum

end

print("What is the length of the array? ")
n = readline()
n = parse(Int, n)
if (n <= 0)
    println("No numbers present in the array!!!")
    exit()
end
arr = Int[]
print("Enter the numbers: ")
arr = [parse(Int, num) for num in split(readline())]
res = maximum_sum_decreasing_subsequence(arr, n)
print("The maximum sum of an  decreasing subsequence of the given array is $res")

"""
Time Complexity: O(num ^ 2), where 'num' is the size of the given array
Space Complexity: O(num)

SAMPLE INPUT AND OUTPUT

SAMPLE 1

What is the length of the array? 10
Enter the numbers: 23 14 5 84 24 222 321 43 123 432
The maximum sum of an  decreasing subsequence of the given array is 444

SAMPLE 2

What is the length of the array? 5
Enter the numbers: 5 4 3 2 1
The maximum sum of an  decreasing subsequence of the given array is 15
"""


using DifferentialEquations
using Plots

sir_ode = @ode_def SIRModel begin
    dS = -β*S*I
    dI = β*S*I-γ*I
    dR = γ*I
    end β γ

parms = [0.1,0.05]
init = [0.99,0.01,0.0]
tspan = (0.0,200.0)

sir_prob = ODEProblem(sir_ode,init,tspan,parms)

sir_sol = solve(sir_prob,saveat = 0.1);

# Plot
plot(sir_sol,xlabel="Time",ylabel="Number")

function sir_ode2(du,u,p,t)
    S,I,R = u
    b,g = p
    du[1] = -b*S*I
    du[2] = b*S*I-g*I
    du[3] = g*I
end

parms = [0.1,0.05]
init = [0.99,0.01,0.0]
tspan = (0.0,200.0)
sir_prob2 = ODEProblem(sir_ode2,init,tspan,parms)
sir_sol = solve(sir_prob2,saveat = 0.1)


### A Pluto.jl notebook ###
# v0.14.7

using Markdown
using InteractiveUtils

# ╔═╡ 2fdecb39-bff4-4f3c-b3f5-35438541a29d
# import necessary libraries
using Random

# ╔═╡ 29802e3a-8dfe-41c7-a25e-2f0961949854
using Statistics

# ╔═╡ cba2ccb0-c5a2-11eb-11be-41eb0278cb72
md"""
### Logistic Regression

[Reference 1](https://towardsdatascience.com/introduction-to-logistic-regression-66248243c148)\
[Reference 2](https://towardsdatascience.com/building-a-logistic-regression-in-python-301d27367c24)\
[ML Glossary](https://ml-cheatsheet.readthedocs.io/en/latest/logistic_regression.html)

#### Sigmoid Function

$$f(x)=\frac{1}{1+e^{-(x)}}$$

which maps predicted values to probabilities

#### Hypothesis Representation

$$Z=\beta_0+\beta_1X$$

$$h\Theta(x)=\text{sigmoid}(Z)=\frac{1}{1+e^{-(\beta_0+\beta_1X)}}$$

#### Cost Function

$$\text{Cost}(h_\theta(x),y)=
    \begin{cases}
    -\log(h_\theta(x)) & \quad \text{if } y = 1\\
    -\log(1-h_\theta(x)) & \quad \text{if } y = 0
    \end{cases}$$

which is also

$$J(\theta)=-\frac{1}{m}\sum \Big[ y^{(i)}\log(h\theta(x^{(i)})) + (1-y^{(i)})\log(1-h\theta(x^{(i)})) \Big]$$

which should be minimized

#### Gradient Descent

$$\theta_j := \theta_j - \alpha\frac{\partial}{\partial\theta_j}J(\theta)$$

In this case:

$$\theta_j := \theta_j - \alpha\sum^m_{i=1}(h_\theta(x^{(i)})-y^{(i)})x_j^{(i)}$$
"""

# ╔═╡ 3761501d-f02e-41cb-9003-74cb68283d74
begin
X_data = rand(Float32, (100, 5))
Y_data = rand(0:1, 100)
size(X_data), size(Y_data)
end

# ╔═╡ ad8d7761-f843-46f4-a7c6-4a5442d1e17a
function sigmoid(Z::Number)::Float32
    k = 1 + MathConstants.e ^ (-Float32(Z))
    return 1 / k
end

# ╔═╡ d457cb21-4e23-4527-a2e4-9654579f714a
typeof(X_data)

# ╔═╡ fcf0ea6c-72c4-40b9-bca9-d1ad40b67dd2
function scale(X::Array)::Array
    @assert ndims(X) == 2
    u = mean(X, dims=1) # compute mean
    s = std(X, dims=1)  # compute standard deviation
    res = (X .- u) ./ s
    return res
end

# ╔═╡ c7a7f2af-fa56-4bbf-afa1-a64b1af5b78d
X_data_scaled = scale(X_data);

# ╔═╡ 1a17435a-484c-4ea8-809b-1d3b0a548288
mean(X_data_scaled, dims=1)

# ╔═╡ 5a87d92e-643f-4032-a6fb-226ce7cab2af
std(X_data_scaled, dims=1)

# ╔═╡ 0890963a-806a-456a-89cf-ad68b11b41c3
# update sigmoid function
function sigmoid(Z::Array)::Array
    denom = 1 .+ (MathConstants.e .^ (-Z))
    return 1 ./ denom
end

# ╔═╡ bf7c00f8-1a1b-4010-b2a9-c02f33351bd5
# test sigmoid function
sigmoid(1234)

# ╔═╡ 4c1f5e23-a985-4f8f-81f5-6b780412a4ad
sigmoid(0)

# ╔═╡ 7fa3c5e2-8431-49d7-94c2-d3885a6a79f6
sigmoid(0.123)

# ╔═╡ ed47d544-9ccd-47ff-bf2a-01a9b2b365b0
# test it
sigmoid(X_data_scaled)

# ╔═╡ 8eaa7843-aa2b-4af3-9e32-629629d0fc36
begin
beta = randn(size(X_data)[2]+1)
size(beta)
end

# ╔═╡ e4d93713-c75d-4cbf-a31c-a9194f25280d
function cost(X::Array, y::Array, beta::Array)::AbstractFloat
    @assert ndims(X) == 2
    @assert ndims(y) == 1
    @assert ndims(beta) == 1
    @assert size(X) == (size(y)[1], size(beta)[1]-1)
    m = size(X)[1]
    X_extended = hcat(X, ones(size(X)[1]))
    X_combined = X_extended * reshape(beta, (size(beta)[1], 1))
    prob = sigmoid(X_combined)
    y_prep = reshape(y, (size(y)[1], 1))
    vec = y_prep .* (log.(prob)) .+ (1 .- y_prep) .* (log.(1 .- prob))
    cost = -1 / float(m) * sum(vec)
    return cost
end

# ╔═╡ 4ae75dea-6fc1-4d10-8c63-176a01193caf
# try it
cost(X_data_scaled, Y_data, beta)

# ╔═╡ 83a969a4-2704-421e-ba02-70875e5b22fc
# prediction function
function predict_proba(X::Array, beta::Array)::Array
    @assert ndims(X) == 2
    @assert ndims(beta) == 1
    @assert size(X)[2] == size(beta)[1]-1
    X_extended = hcat(X, ones(size(X)[1]))
    X_combined = X_extended * reshape(beta, (size(beta)[1], 1))
    prob = sigmoid(X_combined)
    prob = [(prob...)...] # flatten to 1d array
    return prob
end

# ╔═╡ a04e19f3-8c20-47b4-b30a-ccc0696bf54b
predict_proba(X_data_scaled, beta)

# ╔═╡ 7d94d627-941c-41fb-8fce-cc0da6972f74
function predict(X::Array, beta::Array)::Array
    @assert ndims(X) == 2
    @assert ndims(beta) == 1
    @assert size(X)[2] == size(beta)[1]-1
    X_extended = hcat(X, ones(size(X)[1]))
    X_combined = X_extended * reshape(beta, (size(beta)[1], 1))
    prob = sigmoid(X_combined)
    prob = [(prob...)...] # flatten to 1d array
    prob = map(m -> m >= 0.5 ? 1 : 0, prob)
    return prob
end

# ╔═╡ 6418946f-eff9-43fd-8654-415fc41579b5
predict(X_data_scaled, beta)

# ╔═╡ 4c6ee096-98f5-48b8-9977-f2485365759c
# inplace learning function (Gradient Descent)
function learn!(X::Array, y::Array, beta::Array, alpha::AbstractFloat)
    @assert ndims(X) == 2
    @assert ndims(y) == 1
    @assert ndims(beta) == 1
    @assert size(X) == (size(y)[1], size(beta)[1]-1)
    predictions = predict_proba(X, beta)
    offset = predictions .- y
    offset = reshape(offset, (size(offset)[1], 1))
    X_extended = hcat(X, ones(size(X)[1]))
    gradients = X_extended' * offset
    gradients = gradients ./ size(X)[1]
    gradients = gradients .* alpha
    beta .= beta .- [(gradients...)...]
    return nothing
end

# ╔═╡ addad88e-317d-4086-a5e4-a9682f80f54a
beta

# ╔═╡ 2c3b2fed-e9aa-44b5-a12d-40281ae487d9
learn!(X_data_scaled, Y_data, beta, 0.01)

# ╔═╡ 63ab48c3-8c92-4935-a4ce-36a7049e5c86
beta

# ╔═╡ c0f88234-87ec-41ca-8a90-df7d89997090
# define the logistic regression function
function train(X::Array, y::Array; learning_rate::AbstractFloat=0.01, max_iter::Integer=10, return_all::Bool=false)::Array
    @assert ndims(X) == 2
    @assert ndims(y) == 1
    @assert size(X)[1] == size(y)[1]
    @assert max_iter >= 0
    beta = Random.randn(size(X)[2]+1)
    res = nothing
    if return_all
        res = reshape(beta, (1, size(beta)[1]))
    else
        res = beta
    end
    for i = 1:max_iter
        if return_all
            beta = learn(X, y, beta, learning_rate)
            res = cat(res, reshape(beta, (1, size(beta)[1])), dims=1)
        else
            learn!(X, y, beta, learning_rate)
            res .= beta
        end
    end
    return res
end

# ╔═╡ addd77a3-6d60-4387-8bad-65c55fc0171e
# accuracy function
function accuracy(y_pred::Array, y_real::Array)::AbstractFloat
    @assert ndims(y_pred) == ndims(y_real) == 1
    @assert size(y_pred) == size(y_real)
    sum = 0
    for (m, n) in zip(y_pred, y_real)
        if m == n
            sum += 1
        end
    end
    return sum / size(y_pred)[1]
end

# ╔═╡ eb2362a6-8d78-4f38-8b53-a1b9a1f54820
accuracy(predict(X_data_scaled, beta), Y_data)

# ╔═╡ acfefff5-53c3-421c-84c8-441007cbba2e
accuracy(predict(X_data_scaled, train(X_data_scaled, Y_data, max_iter=100, learning_rate=0.5)), Y_data)

# ╔═╡ 8d99210c-eabe-4fe7-a529-4e83be5e6cf8
md"""
Since it is linear model, we should not expect it to have a high accuracy on a completely randomly generated dataset\
But we can see the improvements
"""

# ╔═╡ Cell order:
# ╟─cba2ccb0-c5a2-11eb-11be-41eb0278cb72
# ╠═2fdecb39-bff4-4f3c-b3f5-35438541a29d
# ╠═3761501d-f02e-41cb-9003-74cb68283d74
# ╠═ad8d7761-f843-46f4-a7c6-4a5442d1e17a
# ╠═bf7c00f8-1a1b-4010-b2a9-c02f33351bd5
# ╠═4c1f5e23-a985-4f8f-81f5-6b780412a4ad
# ╠═7fa3c5e2-8431-49d7-94c2-d3885a6a79f6
# ╠═d457cb21-4e23-4527-a2e4-9654579f714a
# ╠═29802e3a-8dfe-41c7-a25e-2f0961949854
# ╠═fcf0ea6c-72c4-40b9-bca9-d1ad40b67dd2
# ╠═c7a7f2af-fa56-4bbf-afa1-a64b1af5b78d
# ╠═1a17435a-484c-4ea8-809b-1d3b0a548288
# ╠═5a87d92e-643f-4032-a6fb-226ce7cab2af
# ╠═0890963a-806a-456a-89cf-ad68b11b41c3
# ╠═ed47d544-9ccd-47ff-bf2a-01a9b2b365b0
# ╠═8eaa7843-aa2b-4af3-9e32-629629d0fc36
# ╠═e4d93713-c75d-4cbf-a31c-a9194f25280d
# ╠═4ae75dea-6fc1-4d10-8c63-176a01193caf
# ╠═83a969a4-2704-421e-ba02-70875e5b22fc
# ╠═a04e19f3-8c20-47b4-b30a-ccc0696bf54b
# ╠═7d94d627-941c-41fb-8fce-cc0da6972f74
# ╠═6418946f-eff9-43fd-8654-415fc41579b5
# ╠═4c6ee096-98f5-48b8-9977-f2485365759c
# ╠═addad88e-317d-4086-a5e4-a9682f80f54a
# ╠═2c3b2fed-e9aa-44b5-a12d-40281ae487d9
# ╠═63ab48c3-8c92-4935-a4ce-36a7049e5c86
# ╠═c0f88234-87ec-41ca-8a90-df7d89997090
# ╠═addd77a3-6d60-4387-8bad-65c55fc0171e
# ╠═eb2362a6-8d78-4f38-8b53-a1b9a1f54820
# ╠═acfefff5-53c3-421c-84c8-441007cbba2e
# ╟─8d99210c-eabe-4fe7-a529-4e83be5e6cf8


"""

    fibonacci(iterations, [num1, num2])

Builds a series of fibonacci numbers `n` iterations in a vector

    Args: iterations: length of fibonacci sequence to be calculated

    Optional: num1 and num2 default to 0 and 1 respectively, the starting numbers of the sequence.

# Examples
```julia-repl

# one argument -  starting values are 0 and 1
julia> fibonacci(10)
12-element Array{Int64,1}:
  0
  1
  1
  2
  3
  5
  8
 13
 21
 34
 55
 89

# two arguments - starting values are 4 and 1
julia> fibonacci(10, 4)
12-element Array{BigInt,1}:
   4
   1
   5
   6
  11
  17
  28
  45
  73
 118
 191
 309

# Three arguments - starting values are 3 and -1
julia> fibonacci(10, 3, -1)
12-element Array{Int64,1}:
  3
 -1
  2
  1
  3
  4
  7
 11
 18
 29
 47
 76
 ```
 """
function fibonacci(iterations, num1=0, num2=1)
    arr = (BigInt)[num1, num2]
    for i in 1:iterations
        push!(arr, +(arr[end], arr[end - 1]))
    end
    return arr
end


using FastGaussQuadrature

"""
    lagrange_1d(points, i, x)

Evaluate the Lagrange interpolation polynomial defined over nodal `points` with index `i`
at point `x`.
"""
function lagrange_1d(points, i, x)
    l = 1.0
    x_i = points[i]
    for x_j in points
        if x_j != x_i
            l *= (x - x_j) / (x_i - x_j)
        end
    end
    return l
end

"""
    lagrange_diff(points, i, x)

Evaluate the derivative of the Lagrange interpolation polynomial defined over nodal `points`
with index `i` at point `x`.
"""
function lagrange_diff(points, i, x)
    dl = 0.0
    x_i = points[i]
    for x_j in points
        if x_j != x_i
            p = 1.0
            for x_m in points
                if (x_m != x_i) & (x_m != x_j)
                    p *= (x - x_m) / (x_i - x_m)
                end
            end
            dl += 1.0/(x_i - x_j) * p
        end
    end
    return dl
end

"""
    get_quadpoints(n)

Compute quadrature points and weights for Gaussian quadrature
of order `n`.
The points (and thus the corresponding weights) are normalized to the range
``[0.0, 1.0]``.

Return a tuple of `points, weights`.
"""
function get_quadpoints(n)
    x, w = gausslegendre(n)
    # They are scaled in [-1.0, 1.0]
    # and we need [0.0, 1.0]
    (x .+ 1)./2, w ./ 2
end

"""
    Basis

A standard 1-dimensional basis of `order::Integer`
with `quadpoints::Array{Float64,1}` and
corresponding `quadweights::Array{Float64,1}`
Is the basis (pun intended) for tensor-product
bases.

    Basis(order::Integer, dimensions)

Initialize a basis of `order::Integer` and
dimensions `dimensions`.
"""
struct Basis
    quadpoints::Array{Float64,1}
    quadweights::Array{Float64,1}
    order::Int64
    dimensions::Int64

    function Basis(order, dimensions)
        quadpoints, quadweights = get_quadpoints(order)
        new(quadpoints, quadweights, order, dimensions)
    end
end

"""
    Base.length(basis::Basis)

Return number of points for `basis` in n-dimensions.
"""
#Base.length(basis::Basis) = length(basis.quadpoints)*basis.dimensions
#Base.length(basis::Basis) = length(basis.quadpoints) ^ basis.dimensions
Base.length(basis::Basis) = basis.order ^ basis.dimensions

"""
    Base.size(basis::Basis)

Return number of points for `basis` for each dimensions as tuple.
"""
function Base.size(basis::Basis)
    ntuple(basis.dimensions) do x
       length(basis.quadpoints)
    end
end

"""
    Base.size(basis::Basis, dim)

Return number of points for `basis` for dimensions `dim`.

function Base.size(basis::Basis, dim::Integer)
    length(basis.quadpoints)
end
"""

function Base.size(basis::Basis, dim::Integer)
    if dim > basis.dimensions
        println("Error: Given Dimension exceeds the Dimension of the problem = ",basis.dimensions)
    else
        length(basis.quadpoints)
    end
end

"""
    evaluate_basis(basis::Basis, coeffs, x)

Evaluate the `basis` with coefficients
`coeffs` at point `x`.

# So the input is coeffs , X point, I need to give basis
#multiplication of the basis is used , indicate tensor product
"""
function evaluate_basis(basis::Basis, coeffs, x)
    #coeffs[1] #code given for order 1

    #To collect result
    basisvalue=0

    # To make linear indexes
    # collect(): makes a array of tuples of indexes, same like meshgrid in matlab
    # vec(): converts matrix into vector
    linearindex = vec(collect(Iterators.product(1:basis.order, 1:basis.order)))

    for i in 1:length(basis)
        basisvalue = basisvalue + ( coeffs[i] * (  lagrange_1d(basis.quadpoints, linearindex[i][1], x[1]) *  lagrange_1d(basis.quadpoints, linearindex[i][2], x[2]) ) )
    end

    return basisvalue
end

"""
    project_to_reference_basis(fun, basis::Basis, ndofs)

Project the result of the function `fun` to coefficients
of the basis built of a tensor-product of `basis`.
The function `fun(x,y)`  takes in the ``x, y``-coordinates
and returns a vector with size `ndofs`.
The corresponding coefficients are returned.

#The reshape() is an inbuilt function in julia which is used
to return an array with the same data as the specified array,
but with different specified dimension sizes.

# It returns the matrix of shape[length(basis),ndof]

"""
function project_to_reference_basis(fun, basis::Basis, ndofs)
    #reshape(fun(0.5, 0.5), (1,ndofs)) #made for constant polynomial of order 1

    # make a matrix of shape [length(basis),ndof]
    coeffarray = zeros( ( length(basis) , ndofs ) )

    # To make linear indexes
    # collect(): makes a array of tuples of indexes, same like meshgrid in matlab
    # vec(): converts matrix into vector

    linearindex = vec(collect(Iterators.product(1:basis.order, 1:basis.order)))

    for i in 1:length(basis)
        coeffarray[i,:] = fun(  basis.quadpoints[linearindex[i][1]]  ,   basis.quadpoints[linearindex[i][2]]  ) # coefficients are nothing but values of function at that point
    end

    return coeffarray # No need to reshaping ,as matrix is made fully and filled it
end

"""
    massmatrix(basis, dimensions)

Return the mass-matrix for a `dimensions`-dimensional
tensor-product basis built up from the 1d-basis `basis`.
"""
function massmatrix(basis, dimensions)
    #ones(1,1)
    mass_matrix = Matrix(1.0I, basis.order^2, basis.order^2)
    ij = 1
    for j = 1:basis.order
        for i = 1:basis.order                                  #not generalising to dimensions>2
            mass_matrix[ij,ij] = basis.quadweights[i] * basis.quadweights[j]                        #eq 4.12 and 4.19
            ij += 1
        end
    end
    mass_matrix

end

"""
    derivativematrix(basis)

Returns the 2-dimensional derivative matrix for `basis`.
Multiplying this with flux-coefficients of shape
`(dimensions * basissize_2d, ndofs)` returns the
coefficients of the corresponding derivative.
"""
function derivativematrix(basis)
    #zeros(1, basis.dimensions * 1)

    # Finding Derivative matrix Try3
    # Finding Dx only, Derivative of the basis function in the direction of x
    Dx = zeros(basis.order^2, basis.order^2)
    Dy = zeros(basis.order^2, basis.order^2)
    linearindex = vec(collect(Iterators.product(1:basis.order, 1:basis.order)))

    for j in 1:basis.order^2
        for p in 1:basis.order^2
            Dx[j,p] = lagrange_diff(basis.quadpoints,linearindex[j][1],basis.quadpoints[linearindex[p][1]]) * lagrange_1d(basis.quadpoints, linearindex[j][2], basis.quadpoints[linearindex[p][2]])
        end
    end

    for j in 1:basis.order^2
        for p in 1:basis.order^2
            Dy[j,p] = lagrange_diff(basis.quadpoints,linearindex[j][2],basis.quadpoints[linearindex[p][2]]) * lagrange_1d(basis.quadpoints, linearindex[j][1], basis.quadpoints[linearindex[p][1]])
        end
    end
    hcat(Dx,Dy)
end

"""
    get_face_quadpoints(basis::Basis, face)

Return the quadrature points at the face `face` for basis `basis`.
"""
function get_face_quadpoints(basis::Basis, face)    # Is there a more efficient/ general way to do this?
    #ones(1.0, 1.0)

    face_quadpoints = zeros(basis.order,2)
    if face == left
        face_quadpoints[:,1] .= 0
        face_quadpoints[:,2] = basis.quadpoints
    elseif face == top
        face_quadpoints[:,1] = basis.quadpoints
        face_quadpoints[:,2] .= 1
    elseif face == right
        face_quadpoints[:,1] .= 1
        face_quadpoints[:,2] = basis.quadpoints
    elseif face == bottom
        face_quadpoints[:,1] = basis.quadpoints
        face_quadpoints[:,2] .= 0
    end
    face_quadpoints
end

function face_projection_matrix(basis, face)
    #ones(1,1)
    face_quadpoints = get_face_quadpoints(basis,face)

    # Matrix of required size
    P_ij = zeros( basis.order  , (basis.order)^2 )

    # Linear indexing
    linearindex = vec(collect(Iterators.product(1:basis.order, 1:basis.order)))

    for j = 1:(basis.order)^2
        for i = 1:basis.order
            P_ij[i,j] = lagrange_1d( basis.quadpoints, linearindex[j][1] , face_quadpoints[i,1] ) *  lagrange_1d( basis.quadpoints, linearindex[j][2], face_quadpoints[i,2] )
        end
    end
    return P_ij
end

"""
    evaluate_m_to_n_vandermonde_basis(basis)

Return the Vandermonde matrix that converts between the 2D-modal
(normalized) Legendre-basis and the 2D-nodal tensor-product basis built
with `basis`.
"""
function evaluate_m_to_n_vandermonde_basis(basis)
    Array{Float64}(I, length(basis), length(basis))
end


using LinearAlgebra
import LinearAlgebra:checksquare

"""
    function sweep_op_internal!(A::AbstractMatrix{Float64}, k::Int64)

    Implement the naive sweep operator described by Goodnight, J. (1979). "A Tutorial on the SWEEP Operator." The American Statistician.
"""
function sweep_op_internal!(A::AbstractMatrix{Float64}, k::Int64)
    p = checksquare(A)
    if k >= p
        throw(ArgumentError("Incorrect k value"))
    end

    @inbounds D = A[k,k] # step 1 D = Aₖₖ

    if D == zero(eltype(A))
        throw(ArgumentError("sweep_op_internal!: the element $k,$k of the matrix is zero. Is the Design Matrix symmetric positive definite?"))
    end

    @inbounds for i in 1:p
        A[k,i] = A[k,i] / D   # step 2 divide row k by D
    end

    @inbounds for i in 1:(k - 1) # step 3: for every other row  i != k
        B = A[i,k]   # let B = Aᵢₖ
        for j in 1:p
            A[i,j] = A[i,j] - B * A[k, j]  # Subtract B * row k from row i
        end
        A[i,k] = - B / D # set Aᵢₖ = -B/D
    end

    @inbounds for i in k + 1:p # step 3: for every other row  i != k
        B = A[i,k]   # let B = Aᵢₖ
        for j in 1:p
            A[i,j] = A[i,j] - B * A[k, j]  # Subtract B * row k from row i
        end
        A[i,k] = - B / D # set Aᵢₖ = -B/D
    end

    @inbounds A[k,k] = 1 / D # step 4 Set Aₖₖ = 1/D

    return A
end

function sweep_op!(A::AbstractMatrix{Float64}, k::Int64)
    sweep_op_internal!(A, k)
end

function sweep_op!(A::AbstractMatrix{Float64}, ks::AbstractVector{Int64})
    for k in ks
        sweep_op_internal!(A, k)
    end
    return A
end

"""
    function sweep_op_full!(A::AbstractMatrix{Float64})

    (internal) Get SSE, error sum of squares, for the full model.
"""
function sweep_op_full!(A::AbstractMatrix{Float64})
    n , p = size(A)

    for k in 1:p-1
        sweep_op_internal!(A, k)
    end
    return A[p,p]
end

"""
    function sweep_op_fullT1SS!(A::AbstractMatrix{Float64})

    (internal) Get SSE, error sum of squares for the full model.
    Also give Type I SS for all independent variables.
"""
function sweep_op_fullT1SS!(A::AbstractMatrix{Float64})
    n , p = size(A)
    TypeISS = Vector{Float64}(undef, p-1)

    for k in 1:p-1
        preSSE = A[p,p]
        sweep_op_internal!(A, k)
        TypeISS[k] = preSSE - A[p,p]
    end
    return A[p,p], TypeISS
end

"""
    function sweep_op_fullT1SS!(A::AbstractMatrix{Float64})

    (internal) Get SSE, error sum of squares for all (p-1) models.
    Also give Type I SS for all independent variables.
"""
function sweep_op_allT1SS!(A::AbstractMatrix{Float64})
    n , p = size(A)
    TypeISS = Vector{Float64}(undef, p-1)
    SSEs = Vector{Float64}(undef, p-1)

    for k in 1:p-1
        preSSE = A[p,p]
        sweep_op_internal!(A, k)
        SSEs[k] = A[p, p]
        TypeISS[k] = preSSE - A[p,p]
    end
    return SSEs, TypeISS
end

"""
    function get_TypeIISS(extended_inverse::AbstractMatrix{Float64})

    (internal) Get Type II SS for all independent variables given an extended inverse (sweep operator already applied)
"""
function get_TypeIISS(extended_inverse::AbstractMatrix{Float64})
    n, p = size(extended_inverse)
    TypeIISS = Vector{Float64}(undef, p-1)

    for k in 1:p-1
        TypeIISS[k] = extended_inverse[k, end]^2 / extended_inverse[k, k]
    end

    return TypeIISS
end


# # An equation of second order in one dimmension
#
# Consider the following problem. Given \\(f \in C([0, 1])\\), find a function \\(u\\) satisfying
# -u'' + u = f in (0, 1)
# u(0) = u(1) = 0

using SymFEL
using SymPy
using LinearAlgebra
using SparseArrays

using PyPlot
close("all")

# discretization parameters
N = 101; # number of nodes
dx = 1 / (N - 1); # discretization step
nodes = range(0, stop=1, length=N);
bound_nodes = [1, N];

# exact solution
u_exact = nodes .* sin.(pi * nodes);
# right hand
f = (1 + pi^2) * nodes .* sin.(pi * nodes) - 2 * pi * cos.(pi * nodes);

# elementary matrices
elem_K = SymFEL.get_lagrange_em(1, 1, 1);
elem_M = SymFEL.get_lagrange_em(1, 0, 0);
elem_K_dx = convert(Matrix{Float64}, elem_K.subs(h, dx));
elem_M_dx = convert(Matrix{Float64}, elem_M.subs(h, dx));

K = SymFEL.assemble_1d_FE_matrix(elem_K_dx, N, intNodes1=0, intNodes2=0, dof1=1, dof2=1);
M = SymFEL.assemble_1d_FE_matrix(elem_M_dx, N, intNodes1=0, intNodes2=0, dof1=1, dof2=1);

F = M * f;

# boundary conditions
tgv = 1e100
A = K + M
A[bound_nodes, bound_nodes] += tgv*sparse(Matrix{Float64}(I, 2, 2));

F[bound_nodes] = zeros(2);

u = A \ F;

plot(nodes, u_exact);
plot(nodes, u);
legend(["Exact solution", "Approximate solution"]);

err = u - u_exact;
println("L2 error = ", sqrt(err' * (M * err)))
println("H1 error = ", sqrt(err' * (K * err)))

# error convergence
NV = 10 * 2 .^ (0:10)
EL2 = zeros(11)
EH1 = zeros(11)
i = 1
for N = NV
    global i
    local dx = 1 / (N - 1); # discretization step
    local nodes = range(0, stop=1, length=N);
    local bound_nodes = [1, N];

    # exact solution
    local u_exact = nodes .* sin.(pi * nodes);
    # right hand
    local f = (1 + pi^2) * nodes .* sin.(pi * nodes) - 2 * pi * cos.(pi * nodes);

    local elem_K_dx = convert(Matrix{Float64}, elem_K.subs(h, dx));
    local elem_M_dx = convert(Matrix{Float64}, elem_M.subs(h, dx));

    local K = SymFEL.assemble_1d_FE_matrix(elem_K_dx, N, intNodes1=0, intNodes2=0, dof1=1, dof2=1);
    local M = SymFEL.assemble_1d_FE_matrix(elem_M_dx, N, intNodes1=0, intNodes2=0, dof1=1, dof2=1);

    local F = M * f;

    # boundary conditions
    local tgv = 1e100
    local A = K + M
    A[bound_nodes, bound_nodes] += tgv*sparse(Matrix{Float64}(I, 2, 2));

    F[bound_nodes] = zeros(2);

    local u = A \ F;

    local err = u - u_exact;

    EL2[i] = sqrt(err' * M * err)
    EH1[i] = sqrt(err' * K * err)

    i += 1
end

figure()
loglog(1 ./ NV, EL2, "d")
loglog(1 ./ NV, EH1)
loglog(1 ./ NV, (1 ./ NV).^2)
xlabel(L"$\Delta x$")
legend(["L2 Error", "H1 Error", L"$\Delta x^2$"])
show()


# Improved Newton's Method

# Safe version - disclosed version

#par1 - lower bound
#par2 - upper bound
function solve_Newton_safely(f,f1,par,par1,par2;nlim = 1e4,ϵ=1e-3)
    for i in 1:nlim
        par3 = par - f(par)/f1(par)
        if ϵ>abs(par3 -par)
            return(par3)
        elseif par3<=par1
            par3 = (par+par1)/2
        elseif par3>=par2
            par3 = (par+par2)/2
        end

        par=par3
    end
    println("Warning! unconverged after" * string(nlim) *"Iterations.")
    return(par3)
end

#where f1 is a constant (avoid finding gradient)
function solve_constGrad(f,f1,par;nlim = 1e4,ϵ=1e-3)
    for i in 1:nlim
        par3 = par - f(par)/f1
        if ϵ>abs(par3 -par)
            return(par3)
        end
        par=par3
    end
    println("Warning! unconverged after" * string(nlim) *"Iterations.")
    return(par3)
end

# by cotangent value
# where
# par1 and par2 are 2 starting points( to give 1st gradient value)
function solve_cotangent(f,par1,par2;nlim = 1e4,ϵ=1e-3)
    for i in 1:nlim
        par3 = par2 - f(par2)*(par2 - par1)/(f(par2) - f(par1))
        if ϵ>abs(par3 -par2)
            return(par3)
        end
        par1 = par2;par2 = par3
    end
    println("Warning! unconverged after" * string(nlim) *"Iterations.")
    return(par3)
end

using Plots

x = 0:.1:4
y=sin.(x)
plot(x,y)
plot!([0,4],[0,0])
solve_cotangent(sin,3,4)


################################
# BIBLIOTECA JULIA CalcNumCTEC #
################################

using LinearAlgebra
using Printf
using Base

function Derivada_MDFCentrada(f::Function, x₀::Number, i::Int; h::Number=10^-4, n::Int=9)
    x = x₀-((n-1)/2)*h:h:x₀+((n-1)/2)*h

    A = zeros(n,n)
    for i in range(1,stop=n)
        for j in range(1,stop=n)
            A[i,j] = x[i]^(n-j)
        end
    end

    coefs = A \ f.(x)

    function dif_f_interp(x_in)
        res = 0
        for k in 1:n-i
            res += coefs[k] * ((factorial(n-k))/(factorial(n-k-i))) * x_in^(n-k-i)
        end
        return res
    end

    return dif_f_interp(x₀)
end

# Cálculo de zeros de funções ------------------------------------------------------------------------------

function Zeros_Bissecao(f::Function, a::Number, b::Number;
    tol::Number=10^-12, klim::Number=10^6)
    if f(a) * f(b) < 0
        k = 0
        x = (a + b) / 2

        while abs(f(x)) > tol
            if k == klim
                @assert false "ERRO: O método não convergiu para $klim iterações..."
            else
                k += 1
            end

            x = (a + b) / 2
            if f(x) * f(a) < 0
                b = x
            else
                a = x
            end
        end

        return x
    else
        @assert false "\n\nERRO NA FUNÇÃO ZeroBissecao:\nOs valores da função nos extremos do intervalo dado\nprecisam ter valores de sinais opostos entre si\npara que a convergência do método seja garantida..."
    end
end

function Zeros_Cordas(f::Function, a::Number, b::Number;
    tol::Float64=10^-12, klim::Int64=10^6)
    if f(a) * f(b) < 0
        k = 0
        x = (a * f(b) - b * f(a)) / (f(b) - f(a))

        while abs(f(x)) > tol
            if k == klim
                @assert false "ERRO: O método não convergiu para $klim iterações..."
            else
                k += 1
            end

            x = (a + b) / 2
            if f(x) * f(a) < 0
                b = x
            else
                a = x
            end
        end

        return x
    else
        @assert false "ERRO NA FUNÇÃO ZeroCordas:\nOs valores da função nos extremos do intervalo dado",
        "\nprecisam ter valores de sinais opostos entre si",
        "\npara que a convergência do método seja garantida..."
    end
end

function Zeros_NR(f::Function, x::Number;
    tol::Float64=10^-12, klim::Int64=10^6)
    k = 0
    while abs(f(x)) > tol
        if k == klim
            @assert false "ERRO NA FUNÇÃO ZeroNR:\nO método não convergiu para $klim iterações..."
        else
            k += 1
        end

        if abs(Derivada_MDFCentrada(f, x, 1, n=5)) < tol
            @assert false "ERRO NA FUNÇÃO ZeroNR:\nO ponto x=$x possui derivada nula..."
        end
        x = x - (f(x) / Derivada_MDFCentrada(f, x, 1, n=5))
    end

    return x
end

# Resolução de sistemas de equações lineares --------------------------------------------------------------

function SEL_EliminGauss(A::Matrix, b::Vector; tol::Float64=10^-12)
    (m,n) = size(A)

    if m==n && m == length(b)
        if abs(det(A)) < tol
            @assert false "ERRO na função EliminGauss:\nA matriz é singular..."
        else
            function test(M)
                aux = 0
                for j in range(1, stop = n)
                    for i in range(1, stop = n)
                        if i > j
                            aux += abs(M[i,j])
                        end
                    end
                end
                return aux
            end

            AG = copy(A)
            bG = copy(b)

            for j in 1:n
                i = j
                while A[j,j] == 0
                    i+=1

                    aux_A = AG[j,:]
                    AG[j,:] = AG[i,:]
                    AG[i,:] = aux_A

                    aux_b = b[j]
                    bG[j] = bG[i]
                    bG[i] = aux_b
                    if (i,j) == (m,n)
                        @assert false "ERRO na função EliminGauss:\nA matriz é singular..."
                    end
                end
            end

            i = 1
            j = 1
            k = 0
            while test(AG) > tol
                k += 1
                if i == n
                    j += 1
                    i = j
                end
                i += 1

                bG[i] -= bG[j] * (AG[i,j] / AG[j,j])
                AG[i,:] -= AG[j,:] * (AG[i,j] / AG[j,j])
            end

            x = zeros(n)
            for k in n:-1:1
                x[k] = (bG[k] - dot(AG[k,:], x)) / AG[k,k]
            end

            return x
        end
    else
        @assert false "ERRO NA FUNÇÃO ElimGauss:\nInconsistência nas dimensões da matriz e do vetor..."
    end
end

function SEL_CritConverg(A::Matrix, modo::Char)
    (m,n) = size(A)

    if m == n
        if uppercase(modo) == 'L'
            beta=zeros(n)
            for i in range(1, stop = n)
                for j in range(1, stop = n)
                    beta[i] += abs(A[i,j])
                end
                beta[i] /= abs(A[i,i])
            end
            return max(beta...)
        elseif uppercase(modo) == 'C'
            beta=zeros(n)
            for j in range(1, stop = n)
                for i in range(1, stop = n)
                    beta[j] += abs(A[i,j])
                end
                beta[j] /= abs(A[j,j])
            end
            return max(beta...)
        elseif uppercase(modo) == 'S'
            beta = ones(n)
            for i in range(1, stop = n)
                for j in range(1, stop = n)
                    beta[i] += abs(A[i,j]) * beta[j]
                end
                beta[i] -= 1
                beta[i] /= abs(A[i,i])
            end
            return max(beta...)
        end
    else
        @assert false "\n\nERRO NA FUNÇÃO CritConverg:\nA matriz dada não é quadrada..."
    end
end

function SEL_GaussJacobi(A::Matrix, b::Vector; x_in::Vector=zeros(length(b)),
    tol::Float64=1e-12, klim::Int=10^6)
    # Rseolve o sistema linear A*x=b através de iterações pelo Método da Gauss-Jacobi.
    # Tolerância padrão: 10^(-12)
    # Métrica de erro: eₐ = ||x_out - x_in|| / ||x_out||
    # Retorno da (x_out, err)

    (m,n) = size(A)

    if m==n && m == length(b)
        if abs(det(A)) < tol
            @assert false "ERRO na função GaussJacobi:\nA matriz é singular..."
        else
            for j in 1:n
                i = j
                while A[j,j] == 0
                    i+=1

                    aux_A = A[j,:]
                    A[j,:] = A[i,:]
                    A[i,:] = aux_A

                    aux_b = b[j]
                    b[j] = b[i]
                    b[i] = aux_b
                    if (i,j) == (m,n)
                        @assert false "ERRO na função EliminGauss:\nA matriz é singular..."
                    end
                end
            end

            function err(x1, x2)
                return norm(x2 - x1) / norm(x2)
            end

            k = 1
            x_out = zeros(n)
            for i in range(1, stop = n)
                x_out[i] = (b[i] - dot(A[i,1:i-1], x_in[1:i-1]) - dot(A[i,i+1:n], x_in[i+1:n])) / A[i,i]
            end

            while err(x_in, x_out) > tol
                if k == klim
                    @assert false "ERRO NA FUNÇÃO GaussJacobi:\nO método não convergiu para $klim iterações\ne tolerância de $tol..."
                else
                    k += 1
                end

                x_in = copy(x_out)

                for i in range(1, stop = n)
                    x_out[i] = (b[i] - dot(A[i,1:i-1], x_in[1:i-1]) - dot(A[i,i+1:n], x_in[i+1:n])) / A[i,i]
                end
            end

            return x_out
        end
    else
        @assert false "ERRO NA FUNÇÃO GaussJacobi:\nInconsistência nas dimensões da matriz e do vetor..."
    end
end

function SEL_GaussSeidel(A::Matrix, b::Vector; x_in::Vector=zeros(length(b)),
    tol::Float64=1e-12, klim::Int=10^6)
    # Rseolve o sistema linear A*x=b através de iterações pelo Método da Gauss-Seidel.
    # Tolerância padrão: 10^(-12)
    # Métrica de erro: eₐ = ||x_out - x_in|| / ||x_out||
    # Retorno da (x_out, err)

    (m,n) = size(A)

    if m==n && m == length(b)
        if abs(det(A)) < tol
            @assert false "\n\nERRO na função GaussJacobi:\nA matriz é singular...\n\n"
        else
            for j in 1:n
                i = j
                while A[j,j] == 0
                    i+=1

                    aux_A = A[j,:]
                    A[j,:] = A[i,:]
                    A[i,:] = aux_A

                    aux_b = b[j]
                    b[j] = b[i]
                    b[i] = aux_b
                    if (i,j) == (m,n)
                        @assert false "ERRO na função GaussSeidel:\nA matriz é singular..."
                    end
                end
            end

            function err(x1, x2)
                return norm(x2 - x1) / norm(x2)
            end

            k = 1
            x_out = zeros(n)
            for i in range(1, stop = n)
                x_out[i] = (b[i] - dot(A[i,1:i-1], x_out[1:i-1]) - dot(A[i,i+1:end], x_in[i+1:end])) / A[i,i]
            end

            while err(x_in, x_out) > tol
                if k == klim
                    @assert false "ERRO NA FUNÇÃO GaussSeidel:\nO método não convergiu para $klim iterações\ne tolerância de $tol..."
                else
                    k += 1
                end

                x_in = copy(x_out)

                for i in range(1, stop = n)
                    x_out[i] = (b[i] - dot(A[i,1:i-1], x_out[1:i-1]) - dot(A[i,i+1:end], x_in[i+1:end])) / A[i,i]
                end
            end

            return x_out
        end
    else
        printstyled("ERRO NA FUNÇÃO GaussJacobi:\nInconsistência nas dimensões da matriz e do vetor...\n\n", color=:bold)
    end
end

# Resolução de sistemas de equações não lineares ---------------------------------------------------------

function SENL_NRSecante(F::Function, x_in::Vector;
    tol::Float64=1e-6, klim::Int64=10^3)
    # Rseolve o sistema não linear F(x)=0 através de iterações pelo Método de Newton-Raphson Secante.
    # Ou seja, a Jacobiana é calculada de maneira aproximada em função da tolerância.
    # Tolerância padrão: 10^(-12)

    function J_aprox(x, δ=tol)
        n = length(x)
        Jap = zeros(n,n)

        for j in range(1, stop = n)
            δx = zeros(n)
            δx[j] = δ * x[j]
            for i in range(1, stop = n)
                Jap[i,j] = (F(x + δx)[i] - F(x)[i]) / δx[j]
            end
        end

        return Jap
    end

    if length(F(x_in)) == length(x_in)
        err(x_in, x_out) = norm(x_out - x_in) / norm(x_out)

        k = 1
        x_out = x_in - J_aprox(x_in) \ F(x_in)

        while err(x_in, x_out) > tol
            if k==klim
                @assert false "\n\nERRO na função NR_Secante:\nO método não convergiu para $klim iterações\ne tolerância de $tol...\n\n"
            else
                k+=1
            end

            x_in = copy(x_out)
            x_out = x_in - J_aprox(x_in) \ F(x_in)
        end

        return x_out
    else
        printstyled("\n\nERRO na função NR_Secante:\nOs vetores possuem dimensões diferentes...\n\n", color=:bold)
        return nothing
    end
end

# Interpolação e ajuste ------------------------------------------------------------------------------

function Interpolacao_Polinomial(x::Vector, y::Vector)
    # Calcula e retorna o polinômio interpolador para os dados de entrada via polinômios de Lagrange.
    # O grau do polinômio é determinado pelas dimensões dos vetores x e y.

    n = Int(length(x))

    if n==length(y)
        function f_interpoladora(x_in)
            res = 0
            for i in range(1, stop=n)
                aux = 1
                for j in range(1, stop=n)
                    if i != j
                        aux *= (x_in - x[j])/(x[i] - x[j])
                    end
                end
                res += y[i] * aux
            end
            return res
        end

        return f_interpoladora
    else
        @assert false "ERRO NA FUNÇÃO Interpolacao_Polinomial:\nOs vetores de entrada possuem dimensões diferentes..."
    end
end

function Interpolacao_Polinomial(f::Function, x::Vector)
    # Calcula e retorna o polinômio interpolador para os dados de entrada via polinômios de Lagrange.
    # O grau do polinômio é determinado pela dimensão do vetor x.

    n = Int(length(x))

    function f_interpoladora(x_in)
        res = 0
        for i in range(1, stop=n)
            aux = 1
            for j in range(1, stop=n)
                if i != j
                    aux *= (x_in - x[j])/(x[i] - x[j])
                end
            end
            res += f(x[i]) * aux
        end
        return res
    end

    return f_interpoladora
end

function Interpolacao_Bilinear(x::Vector, y::Vector, z::Vector)
    if length(x) == length(y) == length(z) == 4
        M = [ones(n) x y x.*y]
        b = M \ z
        g(x,y) = dot(b,[1 x y x*y])
        return g
    else
        @assert false "\n\nERRO NA FUNÇÃO Interpolacao_Bilinear:\nOs vetores de entrada precisam ter dimensões de 4 elementos..."
    end
end

function Ajuste_MinimosQuadrados(x::Vector, y::Vector; grau::Int64=1)
    # Calcula e retorna o polinômio ajustado para os dados de entrada via Método dos Mínimos Quadrados.
    # O grau padrão do polinômio é 1.
    # Retorno da função: (polinomio, r²)
    if length(x) != length(y)
        @assert false "ERRO NA FUNÇÃO MinimosQuadrados:\nOs vetores de entrada possuem dimensões diferentes..."
    else
        A = zeros(grau+1, grau+1)
        for i in 0:grau
            for j in 0:grau
                A[i+1,j+1] = sum(x.^(i+j))
            end
        end

        b = zeros(grau+1)
        for i in 0:grau
            b[i+1] = sum(y.*(x.^i))
        end

        coefs = A \ b
    end

    function f_ap(x_in)
        xᵏ = []
        for i in 0:grau
            push!(xᵏ, x_in^i)
        end
        return dot(coefs, xᵏ)
    end

    Sr = sum((y - f_ap.(x)).^2)
    ym = sum(y)/length(y)
    St = sum((y .- ym).^2)
    r² = 1 - (Sr/St)

    return f_ap, r²
end

# Integração Numérica ------------------------------------------------------------------

# OBS: COMPLETAR O ERRO COM A DIFERENCIAÇÃO NUMÉRICA

function Integral_Trapezio(f::Function, x::Tuple; tol::Float64 = 1e-12)
    if length(x) == 2
        a = x[1]
        b = x[2]
        c = (a+b)/2

        T(a,b) = (b-a) * (f(a) + f(b)) / 2

        if T(a,b) - T(a,c) - T(c,b) < tol
            return T(a,b)
        else
            return Integral_Trapezio(f, (a,c), tol=tol) + Integral_Trapezio(f, (c,b),tol=tol)
        end
    else
        @assert false "ERRO NA FUNÇÃO Intergral_Trapezio:\nO intervalo precisa ter apenas 2 valores..."
    end
end

function Integral_Trapezio(x::Vector, y::Vector)
    n = length(x)

    if n == length(y)
        I = 0
        for i in 1:n-1
            I += ((x[i+1]-x[i])/2)*(y[i] + y[i+1])
        end

        #err = abs(-(b-a)^4 * (IntegSimpson(dif_f4, a, b, n=10^3))[1] / (180*n^4))

        return I#, err
    else
        @assert false "ERRO NA FUNÇÃO Integral_Trapezio:\nOs vetores possuem dimensões diferentes..."
    end
end

function IntegralDupla_Trapezio(f::Function, x::Tuple, y::Tuple; tol::Float64=1e-12)
    if length(x) == length(y) == 2
        x₀, x₁ = x
        y₀, y₁ = y
        xₘ = (x₀ + x₁) / 2
        yₘ = (y₀ + y₁) / 2

        function T(x,y)
            return Integral_Trapezio([y...], [Integral_Trapezio([x...], f.([x...], [y[1],y[1]])),Integral_Trapezio([x...], f.([x...], [y[2],y[2]]))])
        end

        if T(x,y) - T((x₀,xₘ),y) - T((xₘ,x₁),y) < tol
            if T(x,y) - T(x,(y₀,yₘ)) - T(x,(yₘ,y₁)) < tol
                println("teste1")
                return T(x,y)
            else
                println("teste2")
                return IntegralDupla_Trapezio(f, x, (y₀,yₘ), tol=tol) + IntegralDupla_Trapezio(f, x, (yₘ,y₁), tol=tol)
            end
        elseif T(x,y) - T(x,(y₀,yₘ)) - T(x,(yₘ,y₁)) < tol
            println("teste3")
            return IntegralDupla_Trapezio(f, (x₀,xₘ), y, tol=tol) + IntegralDupla_Trapezio(f, (xₘ,x₁), y, tol=tol)
        else
            println("teste4")
            return IntegralDupla_Trapezio(f, (x₀,xₘ), (y₀,yₘ), tol=tol) + IntegralDupla_Trapezio(f, (xₘ,x₁), (y₀,yₘ), tol=tol) + IntegralDupla_Trapezio(f, (x₀,xₘ), y, tol=tol) + IntegralDupla_Trapezio(f, (xₘ,x₁), y, tol=tol)
        end
    else
        @assert false "ERRO NA FUNÇÃO IntegralDupla_Trapezio:\nOs intervalos precisam ter apenas 2 valores..."
    end
end

function Integral_Simpson(f::Function, x::Tuple; tol::Float64 = 1e-12)
    if length(x) == 2
        a = x[1]
        b = x[2]
        c = (a+b)/2

        S(a,b) = (b-a) * (f(a) + 4*f((a+b)/2) + f(b)) / 6

        if S(a,b) - S(a,c) - S(c,b) < tol
            return S(a,b)
        else
            return Integral_Simpson(f, (a,c), tol=tol) + Integral_Simpson(f, (c,b), tol=tol)
        end
    else
        @assert false "ERRO NA FUNÇÃO Integral_Simpson:\nO intervalo precisa ter apenas 2 valores..."
    end
end

function Integral_Simpson(x::Vector, y::Vector)
    n = length(x)

    if n == length(y)
        if n%2==1
            I = 0
            for i in 1:2:n-2
                I += ((x[i+2]-x[i])/6)*(y[i] + 4*y[i+1] + y[i+2])
            end

            return I
        else
            @assert false "ERRO NA FUNÇÃO Integral_Simpson:\nO número de elementos dos vetores deve ser ímpar..."
        end
    else
        @assert false "ERRO NA FUNÇÃO IntegSimpson:\nOs vetores possuem dimensões diferentes..."
    end
end

# function IntegralDupla_Simpson(f::Function, x1::Number, x2::Number,
#     y1::Number, y2::Number; nx::Number=(b-a)*10^3, ny::Number=(d-c)*10^3)

#     hx = (x2-x1)/2nx
#     hy = (y2-y1)/2ny

#     M = zeros(2nx+1, 2ny+1)
#     for (j,y) in enumerate(y1:hy:y2)
#         for (i,x) in enumerate(x1:hx:x2)
#             M[i,j] = f(x,y)
#         end
#     end

#     V = zeros(2ny+1)
#     for k in 1:2ny+1
#         V[k] += Integral_Simpson(Vector(x1:hx:x2), M[:,k])
#     end

#     return Integral_Simpson(Vector(y1:hy:y2), V)
# end

# function IntegralDupla_Simpson(f::Function, x₀::Number, x₁::Number, y₀::Number, y₁::Number; tol::Float64=1e-12)
#     S(a,b) = (b-a) * (f(a) + 4*f((a+b)/2) + f(b)) / 6

#     xₘ = (x₀ + x₁) / 2
#     yₘ = (y₀ + y₁) / 2

#     if

#     else

#     end
# end

function IntegralTripla_Simpson(f::Function, x1::Number, x2::Number, y1::Number, y2::Number,
    z1::Number, z2::Number; nx::Number=(x2-x1)*100, ny::Number=(y2-y1)*100, nz::Number=(z2-z1)*100)

    hx = (x2-x1)/2nx
    hy = (y2-y1)/2ny
    hz = (z2-z1)/2nz

    I = []

    M = Matrix{Number}(undef, 2ny+1, 2nz+1)
    for (k,z) in enumerate(z1:hz:z2)
        for (j,y) in enumerate(y1:hy:y2)
            for (i,x) in enumerate(x1:hx:x2)
                M[i,j] = f(x,y,z)
            end
        end
        # println(M)
        V = zeros(2nx+1)
        for k in 1:2nx+1
            V[k] = Integral_Simpson(Vector(x1:hx:x2), M[:,k])
        end
        # println(V)
        push!(I, Integral_Simpson(Vector(y1:hy:y2), V))
    end
    res = Integral_Simpson(Vector(z1:hz:z2), I)
    return res

    # M = Matrix{Number}(undef, 2nx+1, 2ny+1)
    # for (j,y) in enumerate(y1:hy:y2)
    #     for (i,x) in enumerate(x1:hx:x2)
    #         M[i,j] = Integral_Simpson(Vector(x1:hx:x2), S[i,j,:])
    #     end
    # end

    # V = zeros(2ny+1)
    # for k in 1:2ny+1
    #     V[k] = Integral_Simpson(Vector(x1:hx:x2), M[:,k])
    # end

    # return Integral_Simpson(Vector(y1:hy:y2), V)
end

function IntegralTripla_Trapezio(f::Function, x1::Number, x2::Number, y1::Number, y2::Number,
    z1::Number, z2::Number; nx::Int=10^3, ny::Int=10^3, nz::Int=10^3)

    hx = (x2-x1)/nx
    hy = (y2-y1)/ny
    hz = (z2-z1)/nz

    I = []

    M = Matrix{Number}(undef, ny+1, nz+1)
    for (k,z) in enumerate(z1:hz:z2)
        for (j,y) in enumerate(y1:hy:y2)
            for (i,x) in enumerate(x1:hx:x2)
                M[i,j] = f(x,y,z)
            end
        end
        # println(M)
        V = zeros(nx+1)
        for k in 1:nx+1
            V[k] = Integral_Trapezio(Vector(x1:hx:x2), M[:,k])
        end
        # println(V)
        push!(I, Integral_Trapezio(Vector(y1:hy:y2), V))
    end
    return Integral_Trapezio(Vector(z1:hz:z2), I)

    # M = Matrix{Number}(undef, 2nx+1, 2ny+1)
    # for (j,y) in enumerate(y1:hy:y2)
    #     for (i,x) in enumerate(x1:hx:x2)
    #         M[i,j] = Integral_Simpson(Vector(x1:hx:x2), S[i,j,:])
    #     end
    # end

    # V = zeros(2ny+1)
    # for k in 1:2ny+1
    #     V[k] = Integral_Simpson(Vector(x1:hx:x2), M[:,k])
    # end

    # return Integral_Simpson(Vector(y1:hy:y2), V)
end

# Geometria -----------------------------------------------------------------------------------

struct Vertex2d
    x::Number
    y::Number

    function Vertex2d(x::Number, y::Number)

    end

    function Base.print(v::Vertex2d)
        print("(", v.x, ", ", v.y, ")")
    end
    function Base.println(v::Vertex2d)
        println("(", v.x, ", ", v.y, ")")
    end
end

struct Vertex3d
    x::Number
    y::Number
    z::Number

    function Vertex3d(v::Vertex2d,z::Number)
        return Vertex3d(v.x, v.y, z)
    end

    function Base.print(v::Vertex3d)
        print("(", v.x, ", ", v.y, ", ", v.z, ")")
    end
    function Base.println(v::Vertex3d)
        println("(", v.x, ", ", v.y, ", ", v.z, ")")
    end
end

"CalcNumCTEC incluída com sucesso!"