"""
[Ceva's theorem](https://en.wikipedia.org/wiki/Ceva%27s_theorem) is a theorem about triangles in
plane geometry. Given a triangle `ABC`, let the lines `AO`, `BO` and `CO` be drawn from the vertices
to a common point `O` (not on one of the sides of `ABC`), to meet opposite sides at `D`, `E` and `F`
respectively.  (The segments `AD`, `BE`, and `CF` are known as
[cevians](https://en.wikipedia.org/wiki/Cevian).) Then, using signed lengths of segments,
```math
BD \\times CE \\times AF = DC \\times EA \\times FB.
```
"""
module Ceva

using Plots
using SymPy
using PlaneGeometry
using PlaneGeometry.GeoPlots

export
    ceva_proof, ceva_draw, ceva_rand, ceva, ceva_check

"""
    ceva_draw(📐, O)

Verify Ceva's Theorem for the triangle `📐`.
"""
function ceva_draw(tri, O)

    plist, elist = ceva(tri, O)
    A, B, C, D, E, F = plist
    push!(plist, O)

    plt = plot(tri, fill=(0, :green), aspect_ratio=:equal, fillalpha= 0.2, leg=false)

    plt = scatter!(shape(plist),
            leg=false,
            aspect_ratio=:equal,
            color=:red,
            series_annotations = text.(['A':'F'..., 'O'], :bottom))

    AD, BE, CF = [Edge(epair...) for epair in zip([A, B, C], [D, E, F])]

    for e in [AD, BE, CF]
        plot!(e, color=:orange, leg=false)
    end

    plt, ceva_check(elist...)
end

"""
    ceva_rand()

Verify Ceva's Theorem for a random triangle.
"""
function ceva_rand()
    pts = rand(0:1//10:1,6);
    A, B, C = [Point(pts[i], pts[i+3]) for i in 1:3]

    # Choose O within ABC so the picture looks nicer
    ptx = [pt.x for pt in [A, B, C]]
    pty = [pt.y for pt in [A, B, C]]
    xmin, xmax = N.([f(ptx...) for f in [min, max]])
    ymin, ymax = N.([f(pty...) for f in [min, max]])

    tri = Triangle(A, B, C)
    AB, BC, CA = edges(tri)

    # Use rejections sampling until the point is in ABC
    while true
        x = rand(xmin:1//100:xmax)
        y = rand(ymin:1//100:ymax)
        O = Point(x, y)

        plist, elist = ceva(tri, O)

        # We are unlucky
        if plist == nothing
            continue
        end

        BD, DC, CE, EA, AF, FB = elist

        if BD > BC || DC > BC || CE > CA || EA > CA || AF > AB || FB > AB
            # This mean O is not in ABC
            continue
        end

        return ceva_draw(Triangle(A, B, C), O)
    end
end

"""
    ceva_proof()

Prove Ceva's Theorem.
"""
function ceva_proof()
    @vars by cx positive=true
    @vars cy ox oy

    A = Point(0, 0); B = Point(0, by); C = Point(cx, cy); O = Point(ox, oy)
    tri = Triangle(A, B, C)

    plist, elist = ceva(tri, O)
    ceva_check(elist...)
end

"""
    ceva(Triangle(A, B, C), O)

Returns `([A, B, C, D, E, F], [BD, DC, CE, EA, AF, FB])`.
"""
function ceva(tri, O)
    A, B, C = vertices(tri)
    AB, BC, CA = edges(tri)
    AO, BO, CO = [Edge(pt, O) for pt in vertices(tri)]
    D, E, F = [concurrent(epair...) for epair in zip([BC, CA, AB], [AO, BO, CO])]

    # This mean no such points can be found. (This is possible)
    if D == nothing || E == nothing || F == nothing
        return nothing, nothing
    end

    elist = [Edge(pts...) for pts in zip([B, D, C, E, A, F], [D, C, E, A, F, B])]
    ([A, B, C, D, E, F], elist)
end

"""
    ceva_check(BD, DC, CE, EA, AF, FB)

Check if the line segments satisfies Ceva's theorem.
"""
function ceva_check(BD, DC, CE, EA, AF, FB)
    eq = Eq((BD*CE*AF)^2, (DC*EA*FB)^2)
    N(simplify(eq))
end

end


### A Pluto.jl notebook ###
# v0.14.8

using Markdown
using InteractiveUtils

# This Pluto notebook uses @bind for interactivity. When running this notebook outside of Pluto, the following 'mock version' of @bind gives bound variables a default value (instead of an error).
macro bind(def, element)
    quote
        local el = $(esc(element))
        global $(esc(def)) = Core.applicable(Base.get, el) ? Base.get(el) : missing
        el
    end
end

# ╔═╡ e6db292c-d2be-11eb-1c25-05ed50a9256c
using FourierTools, TestImages, ImageShow, Plots, Colors, FFTW, Noise, PlutoUI

# ╔═╡ 4958079f-da85-4357-94df-b122cf01c958
FFTW.set_num_threads(12)

# ╔═╡ 2a47b7a9-7767-44a7-8dda-480ba1b41f57
md"$\mathcal{F}[U](\mathbf k) = \frac{1}{\sqrt{2\pi}} \int_{-\infty}^{\infty} U(\mathbf r) \cdot \exp[-i  (\mathbf x \cdot \mathbf k)] \, \mathrm d \mathbf x$

$\mathcal{F}^{-1}[\tilde U](\mathbf x) = \frac{1}{\sqrt{2\pi}} \int_{-\infty}^{\infty} U(\mathbf r) \cdot \exp[i  (\mathbf x \cdot \mathbf k)] \, \mathrm d \mathbf k$
";

# ╔═╡ b71a5e39-0805-4e3f-aa83-c3972f6c54af
begin
	img = Float32.(testimage("fabio_gray"));
	img_1D = img[:, 200]
end;

# ╔═╡ c39440fc-17e6-440c-9e10-7b2c9a7e9331
md"### FourierTools.jl | Working with the Frequency Space

* **Felix Wechsler**: Leibniz Institute of Photonic Technology in Jena, Germany
* **Rainer Heintzmann**: Leibniz Institute of Photonic Technology in Jena, Germany
"

# ╔═╡ 6e445cd5-6e51-4a8f-ad5c-2636b3da68f4
md"## FourierTools.jl

* A package built on FFTW.jl
* Provides several helper functions
* implements algorithms (like interpolation) based on FFTs
"

# ╔═╡ e28dc936-04da-48cb-b324-73632d008290
md"
### Definition of Fourier Transform

Complex valued sum

$\mathcal F[x_k] = X_k = \sum_{n=0}^{N-1} x_n \cdot \exp\left[-\frac{i 2 \pi}{N} k n\right] =$

$=\sum_{n=0}^{N-1} x_n \cdot \left[ \cos\left(\frac{2 \pi}{N} k n\right) - i \cdot \sin\left(\frac{2 \pi}{N} k n\right)\right]$

Inverse transform

$\mathcal F^{-1}[X_k] = x_k = \frac{1}{N} \sum_{k=0}^{N-1} X_k \cdot \exp\left[\frac{i 2 \pi}{N} k n\right]$

* it decomposes a signal into harmonic basis functions
* the signal is represented in the _Fourier_ or _Frequency_ space

### Fast Fourier Transform - FFT
* naive sum calculation takes $\mathcal O(N^2)$ operations for a 1D dataset with $N$ points
* FFT algorithms evaluates this with $\mathcal O(N \cdot \log(N))$ operations
* In Julia available _FFTW.jl_
"

# ╔═╡ e8f1e951-26dd-4c0e-83f9-4a2c60b0ebed
md" ## FFT of an dataset with FFTW.jl"

# ╔═╡ 3adab086-0c77-40e5-990a-6b89b183cc4d
begin
	x = range(0, 50, length=999)
	f(x) = (0.1 * sin(17.3 + x) + 0.5 * cos(1.2*x) + 0.3 * sin(5*x)^2 + 0.05 * sin(13.1*x) + sin(27.2 * x)) * exp(-x/20)
	data = add_gauss(f.(x), 0.1)
end;

# ╔═╡ 850869e6-32da-48cf-b2a0-081cc9f8e524
plot(x, data, title="function to transform")

# ╔═╡ d6f9bd7a-ddcb-40b5-8509-f4dac3f8bd5d
begin
	plot(real(fft(data)), title="FFT of the function", xlabel="frequency", label="real part")
	plot!(imag(fft(data)), label="imaginary part")
end

# ╔═╡ b4b57f32-e405-4463-b0fa-625e9d54df34
md"
* Data: $[x_1, x_2, ..., x_N]$
* FFT of Data: $[f_0, f_1, ..., f_{N/2}, f_{-N/2}, f_{-N/2+1}, ...,f_{-1}]$
* (for real data negative and positive frequencies are the complex conjugate)
"

# ╔═╡ f0746bc6-f50a-49f1-96d2-0379498a0168
begin
	plot(fftshift(fftfreq(size(data, 1))), real(ffts(data)), xlabel="frequency", label="real part", title="FFT of the dataset but shifted")
	plot!(fftshift(fftfreq(size(data, 1))), imag(ffts(data)), label="imaginary part")
end

# ╔═╡ 979d641e-83ad-4484-bc28-912947711936
fftshift(fft(data)) == ffts(data)

# ╔═╡ 0ba41c40-21a3-4e26-99fe-cacd76d47a67
typeof(ffts(data))

# ╔═╡ cc160200-5911-4c80-b651-a54e1608f186
begin
	N_low = 64
	x_min = 0.0
	x_max = 8*2π

	xs_low = range(x_min, x_max, length=N_low+1)[1:N_low]
	xs_high = range(x_min, x_max, length=5000)[1:end-1]
	f2(x) = sin(0.5*x) + cos(x) + cos(2 * x) + sin(0.25*x)
	arr_low = f2.(xs_low)
	arr_high = f2.(xs_high)
end;

# ╔═╡ 7acf4c37-397d-4c1f-8735-022888390cbe
md"## Different FFT methods
* `fft` interpretes the data as top-left centered

"

# ╔═╡ b15df267-6606-4583-8ce9-3dddf1ecf3f9
fft([1.0,0,0,0,0])

# ╔═╡ 76bb0c74-1a52-47ff-adf2-862cfea15da5
fft([0,0, 1.0 ,0,0]) # single delta peak

# ╔═╡ ec839c01-a62f-46eb-8e25-15d04a07a79f
ft([0,0, 1.0 ,0,0])

# which is equivalent to fftshift(fft(ifftshift([0,0, 1.0 ,0,0])))

# ╔═╡ c7f0e20e-7c80-4a47-abdf-800f26ca999a
ffts([0,0, 1.0 ,0,0])

# which is equivalent to fftshift(fft([0,0, 1.0 ,0,0]))

# ╔═╡ a725ae98-e2cc-439c-9cc1-692972ac98e7
md"## Calculate sinc interpolation

* Fourier coefficients represent coefficients of Fourier series
* Inverse transform is essentially a re-evaluation of a Fourier Series
* Evaluating this series at more samples points results in an interpolated signal
"

# ╔═╡ d5717c9c-2c16-47da-8533-a28acf189e55
begin
	N = 1000
	xs_interp = range(x_min, x_max, length=N+1)[1:N]
	arr_interp = resample(arr_low, N)
end;

# ╔═╡ 3d8f11a3-4ebf-4d36-a227-b2cafa13e1d6
begin
	scatter(xs_low, arr_low, legend=:bottomleft, markersize=2, label="Low sampling")
	plot!(xs_interp, arr_interp, label="FFT based sinc interpolation", linestyle=:dash)
	plot!(xs_high, arr_high, linestyle=:dashdotdot, label="High sampling")
end

# ╔═╡ 551f179b-46ad-425f-9dd6-d8f6469a09cf
[Gray.(img) Gray.(resample(img, (256, 712)))]

# ╔═╡ a5977c5c-dbcd-402d-b7a3-56614968ccb7
begin
	N_ds = 32
	xs_ds = range(x_min, x_max, length=N_ds+1)[1:N_ds]
	arr_ds = real(resample(arr_high, (N_ds,)))
end;

# ╔═╡ 7f3be670-cfe3-4e60-b38f-ebacb4eaf8d3
md"## Fourier Shift Theorem

$\mathcal{F}[U(x+ Δx)](\mathbf k) = \mathcal{F}[U(x)](\mathbf k) \exp(i k \Delta x)$

$\Longrightarrow$
$U(x+ Δx) = \mathcal{F}^{-1}\bigg[\mathcal{F}[U(x)](\mathbf k) \exp(i k \Delta x)\bigg](x)$

* which allows for efficient sub-pixel shifting
"

# ╔═╡ 1ab61f99-799f-4c34-b0d1-205c01c20dbc
@bind offset Slider(-10:0.01:10, show_value=true)

# ╔═╡ 55667950-4542-4505-9d21-e44ea8b6074d
begin
	f3(x) = cos(4π * x / 30)
	x1 = 1:30
	x2 = x1 .+ 3
	y1 = f3.(x1)
	y2 = f3.(x2)
	y3 = shift(y2, tuple(offset))
end;

# ╔═╡ 8fc68dfc-0fb7-4a5a-bef1-ae0b33ddf4f6
shift(y2, (offset,))

# ╔═╡ cfb02910-fb2d-4de0-bc81-8ecea50e5cb5
begin
	plot(y1, label="Original signal")
	plot!(y2, label="Shifted signal")
	plot!(y3, label="Fourier shifted with $offset")
end

# ╔═╡ 105344de-8617-47fd-aeed-84102f19eaeb
md"## Based on Shifting $\Longrightarrow$ Rotation

"

# ╔═╡ 344f9221-4213-4540-bb40-8a2de2d9b48c
begin
	z = zeros(Float32, round.(Ref(Int), 1.5 .* size(img)))
	FourierTools.center_set!(z, img)
end;

# ╔═╡ 27a7bcae-e50f-4225-bd7f-45934ec55be3
@bind ϕ Slider(-180:1:180, show_value=true)

# ╔═╡ ac331005-48ca-4be8-b0a8-85f6f740f09a
FourierTools.rotate(z, ϕ);

# ╔═╡ bf2c54f1-8110-4e25-b89c-7276cf55f42a
Gray.(FourierTools.rotate(z, ϕ))

# ╔═╡ a5ea8226-ff14-432c-974e-68e1a79155cf
Gray.(FourierTools.rotate(img, ϕ))

# ╔═╡ e1600b0a-b8ef-49b8-8ed0-f9c4ae0c436b

# ╔═╡ 7ff8de2d-85fd-402a-88eb-16427e1e0b47
md"## Convolution
Efficient FFT convolutions for both real and complex functions are included as well
"

# ╔═╡ 03effebd-e918-4847-b13a-7d41d4f159e7
md"Blurring strength"

# ╔═╡ e9862df6-8d1a-4c5e-88ab-1da28c2652af
@bind pow Slider(1:0.01:6, show_value=true)

# ╔═╡ bf631ded-b54c-4768-9a8d-825ab7612695
begin
	x3 = fftpos(1, size(img, 1))
	kernel = exp.(.- 10.0 .^pow .* (x3 .^2 .+ x3'.^2))
	kernel ./= sum(kernel)
end;

# ╔═╡ f4870819-9db6-4bdc-8c8d-141c8fa67461
img_blurry = conv_psf(img, kernel);

# ╔═╡ 9d22e799-fe80-47ef-a326-1f5f1ccc07f8
[Gray.(img) Gray.(img_blurry)]

# ╔═╡ c530d7ea-60bd-4458-b3ca-80b509fb4f33
md"# Conclusion

* FourierTools.jl provides convenient wrappers to the FFTW library
* offers efficient algorithms like sinc interpolation based on FFTs
* deals with the intricacies of the correct handling of even/odd sized arrays in Fourier space
"

# ╔═╡ Cell order:
# ╠═e6db292c-d2be-11eb-1c25-05ed50a9256c
# ╠═4958079f-da85-4357-94df-b122cf01c958
# ╟─2a47b7a9-7767-44a7-8dda-480ba1b41f57
# ╟─b71a5e39-0805-4e3f-aa83-c3972f6c54af
# ╟─c39440fc-17e6-440c-9e10-7b2c9a7e9331
# ╟─6e445cd5-6e51-4a8f-ad5c-2636b3da68f4
# ╟─e28dc936-04da-48cb-b324-73632d008290
# ╟─e8f1e951-26dd-4c0e-83f9-4a2c60b0ebed
# ╟─3adab086-0c77-40e5-990a-6b89b183cc4d
# ╟─850869e6-32da-48cf-b2a0-081cc9f8e524
# ╟─d6f9bd7a-ddcb-40b5-8509-f4dac3f8bd5d
# ╟─b4b57f32-e405-4463-b0fa-625e9d54df34
# ╟─f0746bc6-f50a-49f1-96d2-0379498a0168
# ╠═979d641e-83ad-4484-bc28-912947711936
# ╠═0ba41c40-21a3-4e26-99fe-cacd76d47a67
# ╟─cc160200-5911-4c80-b651-a54e1608f186
# ╟─7acf4c37-397d-4c1f-8735-022888390cbe
# ╠═b15df267-6606-4583-8ce9-3dddf1ecf3f9
# ╠═76bb0c74-1a52-47ff-adf2-862cfea15da5
# ╠═ec839c01-a62f-46eb-8e25-15d04a07a79f
# ╠═c7f0e20e-7c80-4a47-abdf-800f26ca999a
# ╟─a725ae98-e2cc-439c-9cc1-692972ac98e7
# ╟─d5717c9c-2c16-47da-8533-a28acf189e55
# ╟─3d8f11a3-4ebf-4d36-a227-b2cafa13e1d6
# ╠═551f179b-46ad-425f-9dd6-d8f6469a09cf
# ╟─a5977c5c-dbcd-402d-b7a3-56614968ccb7
# ╟─7f3be670-cfe3-4e60-b38f-ebacb4eaf8d3
# ╟─1ab61f99-799f-4c34-b0d1-205c01c20dbc
# ╠═8fc68dfc-0fb7-4a5a-bef1-ae0b33ddf4f6
# ╟─55667950-4542-4505-9d21-e44ea8b6074d
# ╟─cfb02910-fb2d-4de0-bc81-8ecea50e5cb5
# ╟─105344de-8617-47fd-aeed-84102f19eaeb
# ╟─344f9221-4213-4540-bb40-8a2de2d9b48c
# ╟─27a7bcae-e50f-4225-bd7f-45934ec55be3
# ╠═ac331005-48ca-4be8-b0a8-85f6f740f09a
# ╟─bf2c54f1-8110-4e25-b89c-7276cf55f42a
# ╟─a5ea8226-ff14-432c-974e-68e1a79155cf
# ╠═e1600b0a-b8ef-49b8-8ed0-f9c4ae0c436b
# ╟─7ff8de2d-85fd-402a-88eb-16427e1e0b47
# ╟─03effebd-e918-4847-b13a-7d41d4f159e7
# ╟─e9862df6-8d1a-4c5e-88ab-1da28c2652af
# ╠═f4870819-9db6-4bdc-8c8d-141c8fa67461
# ╠═9d22e799-fe80-47ef-a326-1f5f1ccc07f8
# ╟─bf631ded-b54c-4768-9a8d-825ab7612695
# ╟─c530d7ea-60bd-4458-b3ca-80b509fb4f33


#=
P36 (**) Determine the prime factors of a given positive integer (2).
Construct a list containing the prime factors and their multiplicity.
Example:
* (prime-factors-mult 315)
((3 2) (5 1) (7 1))
Hint: The problem is similar to problem P13.
=#

SMALLEST_PRIME = 2

# From P35
function next_prime(n)
    # Brute-force incremental check
    isprime(n) ? n : next_prime(n + 1)
end

# From P35
function _prime_factors(n, x, pf)
    if isprime(n)
        vcat(pf, n)
    else
        if mod(n, x) == 0
            _prime_factors(div(n, x), SMALLEST_PRIME, vcat(pf, x))
        else
            _prime_factors(n, next_prime(x + 1), pf)
        end
    end
end

# From P35
function prime_factors(n)
    # Starting with the smallest prime, 2
    isprime(n) ? n : _prime_factors(n, SMALLEST_PRIME, [])
end

function _prime_factors_multi(pf, n, count, counters)
    if isempty(pf)
        vcat(counters, Vector{}[[n, count]])
    else
        if pf[1] == n
            _prime_factors_multi(pf[2:end], n, count + 1, counters)
        else
            new_counters = vcat(counters, Vector{}[[n, count]])
            _prime_factors_multi(pf[2:end], pf[1], 1, new_counters)
        end
    end
end

function prime_factors_multi(n)
    pf = prime_factors(n)
    _prime_factors_multi(pf[2:end], pf[1], 1, Vector{}[])
end

@assert prime_factors_multi(315) == Vector{}[[3, 2], [5, 1], [7, 1]]

println("Tests passed: JL-36.jl")


"""Julia program to implement Binary Search algorithm.
Binary Search is a Divide and Conquer strategy based algorithm, where the entire array is divided into two halfs and the
required element is searched for in one of these sections as per the conditions. It is done on sorted arrays.
"""

function binary_search(arr, n, ele)
    low = 1
    high = n
    while high >= low
        mid = low + (high - low) ÷ 2
        # If the mid element is the required element, return that index
        if(arr[mid] == ele)
            return mid
        # ele is greater than mid element then ele would be present in first half of the array
        elseif (arr[mid] > ele)
            high = mid - 1
        #Else if  ele is smaller than mid element then ele would be present in last half of the array
        else
            low = mid + 1
        end
    end
    # If the element is not found return 0
    return 0
end

print("How many numbers are present in the array? ")
n = readline()
n = parse(Int, n)
if (n <= 0)
    println("Array is Empty!!!")
    exit()
end
arr = Int[]
print("Enter the numbers: ")
arr = [parse(Int, num) for num in split(readline())]
print("Which number do you want to search in the array? ")
ele = readline()
ele = parse(Int, ele)
# Sort the array in ascending order
arr = sort(arr)
res = binary_search(arr, n, ele)
if (res == 0)
    print("The number $ele is not present in the array")
else
    print("The number $ele is present in the array.")
end

"""
Time Complexity - O(log(n)), where 'n' is the size of the array
Space Complexity - O(n)

SAMPLE INPUT AND OUTPUT

SAMPLE I

How many numbers are present in the array? 5
Enter the numbers: 1 2 3 4 5
Which number do you want to search in the array? 6
The number 6 is not present in the array

SAMPLE II

How many numbers are present in the array? 3
Enter the numbers: 3 1 2
Which number do you want to search in the array? 2
The number 2 is present in the array.

"""


#****************************************************************#
# VeysiADN 4 Apr 2021
# Advanced Robotics Homework 3 Codes
#****************************************************************#
using LinearAlgebra
#****************************************************************#
# Returns 3 x 3 rotation matrix representing
# rotation about unit vector w by angle theta.
# theta should be in radian
function Rodrigues(w,theta)
    if( length(w) != 3)
        println("!Warning! : W matrix should have 3 elements.")
        return
    end
    if(size(w)==(1,3))
        w=w';
    end
    w_hat = [0      -w[3]    w[2];
             w[3]     0     -w[1];
            -w[2]    w[1]      0];
    rotation_matrix = cos(theta)*Matrix(1I,3,3) + sin(theta)*w_hat+(1-cos(theta))*w*w';
    return rotation_matrix;
end
#****************************************************************#
# Builds rigid transformation matrix T from given
#rotation matrix R and translation t
function RpToTrans(R,t)
    if( size(R)!=(3,3) )
        println("!Warning! : Rotation matrix should be 3x3.")
        return
    end
    if( length(t) != 3 )
        println("!Warning! : Translation matrix should have 3 elements.")
        return
    end
    if(size(t)==(1,3))
        t=t';
    end
    R_B_T = hcat(R,t) ;
    R_B_T = vcat(R_B_T,[0 0 0 1]) ;
    return R_B_T
end
#****************************************************************#
#Extracts rotation matrix R and translation t from given /
# rigid transformation matrix T
function TransToRp(T)
    if( size(T)!=(4,4) )
        println("!Warning! : Transformation matrix should be 4x4.")
        return
    end
    R = T[1:3,1:3]
    t = T[1:3,4]
    return R,t;
end
#****************************************************************#
# Inverts rigid transformation matrix T
function TransInv(T)
    if( size(T)!=(4,4) )
        println("!Warning! : W matrix should be 4x4.")
        return
    end
    if( det(T)==0 )
        println("!Warning! : Inverse doesn't exist (Determinant is equal to zero).")
        return
    end
    R,t = TransToRp(T);
    r_inv =hcat(R',-(R'*t))
    r_inv = vcat(r_inv,[0 0 0 1]);
    return r_inv;
end
#****************************************************************#


module NewtonsMethod

using LinearAlgebra, Statistics, Compat, ForwardDiff

function newtonroot(f, f_prime; x_0, tolerance = 1E-7, maxiter = 1000)
    x_old = x_0
    error = Inf
    iter = 1
    while error > tolerance && iter <= maxiter
        if f_prime(x_old) ≈ 0.0
            return (root = nothing, error = nothing, iter = nothing)
        end
        x_new = x_old - f(x_old)/f_prime(x_old)
        error = norm(x_new - x_old)
        x_old = x_new
        iter = iter + 1
    end
    if iter == maxiter+1
        return (root = nothing, error = nothing, iter = nothing)
    else
        return (root = x_old, error = error, iter = iter)
    end
end

# Applying auto-differentiation
D(f) = x -> ForwardDiff.derivative(f, x)

newtonroot(f; x_0, tolerance = 1E-7, maxiter = 1000) = newtonroot(f, D(f), x_0 = x_0, tolerance = tolerance, maxiter = maxiter)

# Export function
export newtonroot

end


#' # Minimizing functions - how a computer learns

#' In the previous notebooks, we've seen that by changing **parameters** in a
#' function, we could find a "best fit" of that function to some data. We use
#' a **loss function** to quantify the "goodness" of a set of parameters and
#' we look for those parameters that **optimize**, in fact **minimize**,
#' the loss function. If we teach a machine how to minimize the loss function
#' on its own, we say that machine is able to **learn** how to model our data.
#'
#' In the last notebook, we threw around terms like **derivative**,
#' **gradient**, and **gradient descent** to give you a rough sense of how we
#' minimize a function on a computer. In this notebook, we will step through
#' these concepts more carefully, with the aim of being able to implement them
#' using Julia.

#' ## Minimizing a 1D function using calculus
#' Let's draw the 1D loss function $L_1$ from a previous notebook again:

using Plots
#gr()
#pyplot()
plotly()

sigma(x) = 1 / (1 + exp(-x))
f(x,w) = sigma(w * x)

x1 = 2
y1 = 0.8

L1(w) = (y1 - f(x1, w))^2

plot(L1, -2, 1.5, xlabel="w", ylabel="L1(w)", legend=false)

#' By eye, we can see that the minimum is around $w=0.6$. But how can we get
#' the computer to work this out on its own?

#' In the previous notebook, we thought of this function plot as a hill,
#' viewed from the side. We could find the minimum by making the hill sticky,
#' and letting a ball roll down it. The ball will find and settle in the
#' minimum of the function.  Now let's see how to teach a computer to do this.

#' We need to find the downhill direction along the hill, which is related to
#' its *slope* (how steep it is). Calculus provides us with tools to calculate
#' that slope!

#' Namely, the slope of a curve $L_1(w)$ at $w$ is given by its **derivative**
#' $L_1'(w)$; geometrically, this is the slope of the **tangent line** to the
#' curve at that point, i.e. the straight line which touches the curve at that
#' point.

#' Calculus provides us with some rules to calculate an analytical formula for
#' the derivative, and we will see later how to apply these rules, albeit
#' indirectly, for machine learning.
#' To gain understanding, however, we will see here how to get the computer
#' to help us out by calculating the derivatives **numerically** instead!

#' ## Approximating derivatives

#' Let's recall that the derivative $L_1'(w)$ of a function is defined as
#'
#' $$L_1'(w) \simeq \frac{L_1(w+h) - L_1(w)}{h},$$
#'
#' for a small step size $h$. (Strictly speaking, we must take the limit
#' when $h$ tends to $0$ to obtain the exact value of the derivative.)

#' #### Exercise 1
#'
#' Write a function to calculate the derivative of a function at a given
#' point. Note that in Julia, we can easily pass functions as arguments
#' to other functions!
#' The function should take the function, the point $w$ at which to calculate
#' the derivative, and the value of $h$, which should have a default value
#' of 0.001.
#'
#' *Note*: A function input argument can have a *default value* if we set
#' the input argument equal to that default value when we define the function.
#' For example,
#'
#' ```julia
#' f(x, a = 3) = a * x^2
#' ```
#'
#' The function `f` will square the input we give it and multiply by `a`.
#' However,  if we choose to call `f(x)` *without* passing it an `a` input,
#' it will assume `a` is `3` and return `3*x^2`.

#' #### Solution
#'
#' We could define the `derivative` function as
function derivative(f, w, h=0.001)
    Ans = f(w + h) - f(w)
    Ans /= h
    return Ans
end

#' **Tests**
@assert isapprox(derivative(x -> x^2, 0.2), 0.4, atol=0.01)
@assert isapprox(derivative(x -> x^2, 0.2, 0.5), 0.9, atol=0.5)
@assert isapprox(derivative(x -> x^3, 0.2), 0.12, atol=0.01)

#' #### Exercise 2
#'
#' Write an interactive visualization of the tangent line to the graph of
#' $L_1$, so that we can visualize the tangent at any point on $L_1$.
#' Include the current value of the derivative in the title.
#'
#' *Hint*: Recall that the straight line through the point $(x_0, y_0)$ with
#' slope $m$ is given by
#'
#' $$\frac{y - y_0}{x - x_0} = m,$$
#'
#' so
#'
#' $$y = y_0 + m*(x - x_0).$$

wmin = -2
wmax = 1
w = 0.5
pl = plot(L1, wmin, wmax, label="L_1")

x0 = w
y0 = L1(w)
scatter!(pl, [x0], [y0], label="")

m = derivative(L1, w)

plot!(pl, x -> y0 + m * (x - x0), ls=:dash, label="tangent line")

ylims!(pl, -0.1, 0.6)
xlims!(pl, wmin, wmax)

#' #### Exercise 3
#'
#' What is the value of the derivative (slope of the tangent line) at a
#' minimum? Can this happen anywhere else?

#' #### Solution
#' The derivative is `0` at a minimum. This happens at any minimum, maximum or
#' "point of inflection" of a smooth function.

#' #### Exercise 4
#'
#' When the derivative $L_1'(w)$ is positive, that means that $L_1$ increases
#' from left to right at point $w$.
#'
#' If the derivative $L_1'(w)$ is positive (> 0), in which direction should
#' we move $w$ to *decrease* $L_1$?

#' #### Solution
#'
#' We should move $w$ left, i.e. decrease it.

#' #### Exercise 5
#'
#' If the derivative $L_1'(w)$ is negative (< 0), in which direction should we
#' move $w$ to *decrease* $L_1$?

#' We can use this information to tell the computer which way to take a step.
# This constitutes the numerical algorithm called **gradient descent**;
#' it is called this since we are descending (moving downwards) along the
#' graph of the function by using information about its gradient (slope).

#' #### Exercise 6
#'
#' Implement gradient descent by following this prescription for an
#' **iterative (repetitive) algorithm**:
#'
#' 1. Start at an initial guess $w_0$ for $w$.
#'
#' 2. At each step, calculate the derivative, $L_1'(w_n)$ at the current value
#' of $w_n$ using the function that you created above.
#'
#' 3. Modify the value of $w$ by a small multiple (for example, $\eta=0.01$)
#' of the value of the derivative you just created, via
#' $w_{n+1} = w_n - \eta L_1'(w_n)$.
#'
#' For this problem, start with $w_0 = -2.0$. Repeat steps 2 and 3 a total
#' of `2000` times.
#'
#' Package this code into a function called `gradient_descent` that takes two
#' inputs, a function and a range for values of $w$, and returns the final
#' value of $w$ and $L_1(w)$.
#'
#' Using `L1` and `-2:0.01:1.5` as your inputs to `gradient_descent`,
#' for what value of $w$ is $L_1$ at a minimum?

#' #### Solution
function gradient_descent(f) # wrange not used?
    w = -2.0
    for i in 1:2000
        fprime = derivative(f, w)
        step = 0.01 * fprime
        w = w - step
    end

    return w, f(w)
end

(w, fw) = gradient_descent(L1)

#' #### Exercise 7
#'
#' Modify your code for gradient descent to return the result once it has
#' found an answer within some *tolerance*, rather than taking a set number
#' of steps. The new prescription for this algorithm is:
#'
#' 1. Start at an initial guess $w_0$ for $w$.
#'
#' 2. At each step, calculate the derivative, $L_1'(w_n)$ at the current value
#' of $w_n$, using the function that you created above.
#'
#' 3. Modify the value of $w$ by a small multiple (for example, $\eta=0.01$)
#' of the value of the derivative you just created, via
#' $w_{n+1} = w_n - \eta L_1'(w_n)$.
#'
#' 4. Check how different $w_{n+1}$ is from $w_n$. If you're satisfied that
#' $L_1(w_{n+1})$ is minimized, return $w_{n+1}$ and $L_1(w_{n+1})$.
#' Otherwise, go to step (2) and continue.
#'
#' Edit `gradient_descent` so that it takes three inputs: a function, a range
#' for values of $w$, and a tolerance that tells you how close $w_{n+1}$ must
#' be to $w_n$ before you can stop iterating.
#'
#' Using `L1`, `-2:0.01:1.5`, and `.000001` as your inputs to
#' `gradient_descent`, for what value of $w$ is $L_1$ at a minimum?

#' #### Solution
#'
#'
#' Your code should look something like this
#'
#' $L_1$ is at a minimum when `gradient_descent(L1, -2:0.01:1.5, .000001)[1]`,
#' or $w$, is around `0.69216`.

function gradient_descent(f, tol)
    w = -2.0
    ccount = 0

    while ccount < 2000000   # do no more than 2000 times
        fprime = derivative(f, w)
        step = 0.01 * fprime
        w = w - step
        if abs(step) < tol
            return w, f(w)
        end

        ccount += 1
    end
end

(w, fw) = gradient_descent(L1, 0.000001)
@assert 0.69215 < w < 0.69217

#' After you've completed this step, you will have found a good approximation
#' to the value $w^*$ where $L_1$ reaches its minimum, and the minimum
#' value $L_1(w^*)$, which in this case is (almost) $0$.

#' #### Exercise 8
#'
#' Alter the function `gradient_descent` so that it stores the results
#' `(w, L1(w))` at each step of the algorithm as an array and returns
#' this array. How many steps does the algorithm take for input parameters
#' `(L1, -2:0.01:1.5, .000001)` before terminating? You should count your
#' starting $w_0$ as your first step.

"""
    gradient_descent(f, wrange, tol)

Use gradient descent method to find the minimum of a function.

# Examples

# Arguments
- `f`: Find the minimum of the function `f`.
- `wrange`: User proivded initial guess range.
- `tol`: Tolerance of consequtive step size.  Returns when the difference of
        consequtive steps is less than `tol`.
"""
function gradient_descent(f, wrange; tol=0.000001, w_init=undef)
    # find initial guess from wrange
    if w_init == undef
        w_vec = Vector{Float64}(undef, 0)
        fw_vec = Vector{Float64}(undef, 0)

        for w = wrange
            fw = f(w)
            push!(w_vec, w)
            push!(fw_vec, fw)
        end

        min_ind = argmin(fw_vec)
        w_init = w_vec[min_ind]
        fw_init = fw_vec[min_ind]
        println("Initial guess: $w_init, $fw_init")
    # return (w_inti, fw_init)
    end

    # gradient descent
    w = w_init
    w_vec = Vector{Float64}(undef, 0)
    fw_vec = Vector{Float64}(undef, 0)
    ccount = 0
    fw = f(w)
    push!(w_vec, w)
    push!(fw_vec, fw)
    while ccount < 1000000
        fprime = derivative(f, w)
        step = 0.01 * fprime

        w = w - step
        fw = f(w)
        push!(w_vec, w)
        push!(fw_vec, fw)
        if abs(step) < tol
            println("iteration = $ccount")
            return w_vec, fw_vec
        end

        ccount += 1
    end

    error("Exceeded maximum iterations")
end
(w, fw) = gradient_descent(L1, -2:0.1:1.5; tol=0.000001)
println("Iteration count: $(length(w))")

#' #### Exercise 9
#'
#' Overlay the steps taken in the last exercise with a plot of
#' $L_1(w)$ vs. $w$.
#'
#' Where does our algorithm take the largest steps?
#' (Where does the ball move fastest down the hill?)
#'
#' A) Between w = -2:-1

#' B) Between w = -1:0

#' C) Between w = 0:.6
#'
#' *Hint*: It may be easier to see what's going on if you only plot,
#' for example, every 15th step.

# Initial guess = 0.7 so the plot range needs to be smaller to see
(w, fw) = gradient_descent(L1, -2:0.1:1.5; tol=0.000001)
pl = plot(L1, 0.692, 0.70, xlabel="w", ylabel="L1(w)", legend=false)
ws = reverse(w[end:-50:1]) # Make sure end point is included
fws = reverse(fw[end:-50:1])
scatter!(pl, ws,fws)

# Initial guess = -2.0
(w, fw) = gradient_descent(L1, -2:0.1:1.5; tol=0.000001, w_init=-2)
pl = plot(L1, -2, 1.5, xlabel="w", ylabel="L1(w)", legend=false)
ws = reverse(w[end:-50:1]) # Make sure end point is included
fws = reverse(fw[end:-50:1])
scatter!(pl, ws,fws)

#' ## Functions of 2 variables and their derivatives
#' So far, we have been looking at the minimizing a loss function
#' $L_1(w)$ that depends on a single parameter, $w$. Now let's turn to
#' the cost function $L_2(w, b)$ from a previous notebook, that is a
#' function of *two* parameters, $w$ and $b$, and try to minimize it.
#'
#' As we've seen, we get a **surface**, instead of a curve,
#' when we graph $L_2$ as a function of both of its parameters.

#' **Exercise 10**
#'
#' Draw a surface plot of $L_2$, given by
#'
#' $$L_{2}(w, b) = \sum_i(y_i - g(x_i, w, b))^2$$
#'
#' using the `surface` function from `Plots.jl`. For this plot, use the
#' values of `xs` and `ys` from notebook 5:
#'
#' We can get a nice interactive 3D plot by using the Plotly backend
#' of `Plots.jl` by executing the command
#'
#'     plotly()
# if !(@isdefined WEAVE_ARGS)
#     plotly()
# end

xs = [2, -3, -1, 1]
ys = [0.8, 0.3, 0.4, 0.4]

sigma(x) = 1/(1 + exp(-x))
g(x,w,b) = sigma(w*x) + b

L2(w,b) = sum(abs2, ys .- g.(xs,w,b))

pl = plot()
surface!(pl, -2:0.02:2, -2:0.02:2, L2, alpha=0.6, xlabel="w",
        ylabel="b", zlabel="L2(w,b)",
        size=(600,600))
#'

#+ echo=false
if !@isdefined WEAVE_ARGS # exe when not from Weave
    gui(pl)
end

#' ### Finding the minimum
#' We can just about see, by rotating the graph, that $L_2$ has a single
#' minimum. We want to find the values of $w$ and $b$ where this
#' minimum value is reached.

#' Following what we did for the function $L_1$ above, we expect that
#' we will need to calculate derivatives of $L_2$. Since the function is
#' more complicated, though, the derivatives are too!
#'
#' It turns out that the right concept is that of the
#' [**gradient**](https://en.wikipedia.org/wiki/Gradient) of
#' $L_2$, denoted $\nabla L_2(w, b)$. This is a **vector** consisting
#' of $2$ numbers if there are $2$ parameters
#'[or, in general, $n$ numbers if there are $n$ parameters].
#'
#' The numbers that form the gradient $\nabla L_2(w, b)$ are called the
#' **partial derivatives** of $L_2$ with respect to $w$ and $b$,  written as
#'
#' $$\frac{\partial L_2}{\partial w} \quad \text{and} \quad \frac{\partial L_2}{\partial b}.$$
#'
#' Although this notation might look complicated, all it means is that we
#' calculate derivatives just like we did before, except that we fix the value
#' of the other variable.
#' For example, to calculate $\frac{\partial L_2}{\partial w}$,
#' the partial derivative of $L_2$ with respect to $w$, we fix the value of
#' $b$ and think of the resulting function as a function of the single
#' variable $w$; then we use the formula for derivatives of functions of a
#' single variable.
#'
#' [Note that $\frac{\partial L_2}{\partial w}$ is itself a function of $w$ and $b$; we could write $\frac{\partial L_2}{\partial w}(w, b)$.]

#' #### Exercise 11
#'
#' Write functions that will allow you to calculate the partial derivatives of
#' a function $f$ of two variables.
#'
#' In particular, declare functions called `partial_w` and `partial_b`.
#' Each should take four inputs - a function $f$ of two variables, the first
#' input argument to $f$, the second input argument to $f$, and a step size
#' `h` with default value `0.001`. `partial_w` should return the partial
#' derivative of $f$ with respect to its first input argument and `partial_b`
#' should return the partial derivative of $f$ with respect to its second
#' input argument.

#' #### Solution
#'
#' The partial derivatives are
partial_w(f, w, b, h=0.001) = (f(w+h,b) - f(w,b))/h
partial_b(f, w, b, h=0.001) = (f(w,b+h) - f(w,b))/h

#' **Test**
partial_x = partial_w
partial_y = partial_b
fn(x,y) = 3*x*y
#' Test default step
@assert isapprox(partial_x(fn, 0.3, 0.3), 0.9; atol=1.e-6)
@assert isapprox(partial_y(fn, 0.3, 0.3), 0.9; atol=1.e-6)
#' Test custom step
fn(x,y) = 3 * x^2 * y^3
@assert isapprox(partial_x(fn, 2.3, 10.2, 1), 17828.2944; atol=1.e-2)
@assert isapprox(partial_y(fn, 2.3, 10.2, 1), 5454.84; atol=1.e-2)

#' #### Exercise 12
#'
#' Use `partial_b` from the last exercise to find the partial derivative of
#' $L_2$ with respect to $w$ at b = 0.3,
#' $\frac{\partial L_2}{\partial w}|_{b = 0.3}$ for `w = -2:0.01:1`

wrange = -2:0.01:1
partialw = partial_w.(L2, wrange, 0.3)

#' #### Exercise 13
#'
#' Plot the cross section of the surface of $L_2(w, b)$ at $b = 0.3$.
#' Make this plot interactive to show that the function `partial_w` gives
#' the slope of the tangent to this cross section for any point `w` in
#' the range `-2:0.01:1`.
#'
#' For what value of $w$ in this range is the slope of the cross section
#' closest to -1?

L2_b(w) = L2(w, 0.3) # b = 0.3

w = 0.0 # Change me
pl = plot(L2_b, wrange) #plot(wrange, L2_b)
scatter!([w], [L2_b(w)])

slope = partial_w(L2, w, 0.3)
wrange_value = slope .* wrange .- slope*w .+ L2.(w,0.3)
plot!(pl, wrange, wrange_value)
ylims!(0.3, 2.0)
title!("Slope = $slope @ $(w)")
#'
#' ## **Optional**: Functions with $n$ inputs
#' If a function $f$ takes $n$ input arguments, we can write them as
#' $p_1, \ldots, p_n$, where $p_i$ means the "$i$th parameter".
#' In Julia, we can wrap them up into a single **vector**. Now we can
#' calculate the partial derivative $\frac{\partial L_2}{\partial p_i}$ with
#' respect to the $i$th variable.

#' #### Exercise 14
#'
#' For the next exercise, you will need to use the splat command, `...`. You can use this command to "open up" a collection and pass all the elements of that collection as inputs to a function.
#'
#' For example, say you have an array, `numbers`,
#'
#' ```julia
#' numbers = [4, 3, 2]
#' ```
#'
#' and you want to use `numbers` to create a $4\times3\times3$ randomly populated array via `rand`. `rand(numbers)` will not do what you want. You could index into `numbers` to grab the values you want and pass them to `rand`, as in
#'
#' ```julia
#' rand(numbers[1], numbers[2], numbers[3])
#' ```
#'
#' or you could use a splat:
#'
#' ```julia
#' rand(numbers...)
#' ```
#'
#' Use `...` to pass the contents of `inputs`
#'
#' ```julia
#' inputs = [30, 12, "cats"]
#' ```
#'
#' to the function `dreams`
#'
#' ```julia
#' dreams(i, j, perfect_mammal) = "I wish I had $(i + j) $perfect_mammal."
#' ```

inputs = [30, 12, "cats"]
dreams(i, j, perfect_mammal) = "I wish I had $(i + j) $perfect_mammal."
dreams(inputs...)

# **Tests**

@assert dreams(inputs...) == "I wish I had 42 cats."

#' #### Exercise 15:
#'
#' Write a function, `partial`, to calculate the $i$th partial derivative of a
#' function. This function should have four inputs
#'
#' * a function, $f$, for which you want to compute the partial derivative
#' * an array, *p*, specifying the values of all input arguments to $f$ at
#'   the point where you want $\frac{\partial f}{\partial p_i}$ computed
#' * the index, $i$, of the variable with respect to which you want to calculate the partial derivative of $f$
#' * a step size with default value 0.001
#'
#' Hint: you will need to `copy` and modify `p` within `partial`.

function partial(fn, p, i, h=0.001)
    p = copy(p)
    a = fn(p...)
    p[i] += 1*h
    b = fn(p...)
    Ans = (b - a)/(1*h)

    return Ans
end

fn1(x,y,z) = x^2 * y * z
fn1(xyz) = fn1(xyz...)
#' $\frac{\partial fn_1}{\partial x}$
fn1px(x,y,z)  = 2*x*y*z
fn1py(x,y,z) = x^2*1*z
fn1pz(x,y,z) = x^2*y*1

x = 1.0
y = 2.0
z = 1.0
xyz = [x, y, z]

actual = partial(fn1, xyz, 1, 1.e-6)
expected = fn1px(xyz...)
@assert isapprox(actual, expected; atol=1.e-4)

using FiniteDifferences
# what the?  Why do I have to use 2nd order central difference for
# first derivate?
actual = central_fdm(2,1)(x->fn1(x,y,z), 1.0)
@assert isapprox(actual, expected; atol=1.e-6)
actual = forward_fdm(2,1)(x->fn1(x,y,z), 1.0)
@assert isapprox(actual, expected; atol=1.e-6)

using FiniteDiff
# This is right
actual = FiniteDiff.finite_difference_derivative(x->fn1(x,y,z), 1.0)
@assert isapprox(actual, expected; atol=1.e-6)

# GradientFiniteDiff.finite_difference_gradient(fn1, xyz)
(gx,gy,gz) = FiniteDiff.finite_difference_gradient(fn1, xyz)

w = 1.1
x = 2.2
y = 3.3
z = 4.4
fn2(w,x,y,z) = w * x^2 * y^3 * z^4
fn2pw(w,x,y,z) = x^2 * y^3 * z^4
fn2px = w * 2 * x * y^3 * z^4
fn2py = w * x^2 * 3 * y^2 * z^4
fn2pz = w * x^2 * y^3 * 4 * z^3

# Test partial()
expected = fn2pw(w,x,y,z)

actual = partial(fn2, [w,x,y,z], 1, 1.e-6)
@assert isapprox(actual, expected; rtol=1.e-6)

actual = FiniteDiff.finite_difference_derivative(
            w->fn2(w,x,y,z), w)
@assert isapprox(actual, expected; rtol=1.e-9)

#' ## Gradient descent in 2 dimensions
#' It turns out that the gradient vector of the function $L_2(w, b)$ gives the
#' direction in the plane $(w, b)$ in which the function $L_2$
#' **increases fastest**.
#'
#' In other words, if you start at the position $(w_0, b_0)$ and take a small
#' step of length $\eta$ to new values $(w', b')$, the value of the function
#' will change to a new value $L_2(w', b')$. How do we get to the minimum of
#' $L_2(w, b)$ as fast as possible? We want to step in the direction where
#' $L_2$ decreases fastest!
#'
#' In multivariable calculus courses, it is shown that $L_2$ will *increase*
#' fastest if you take a step **in the direction of the gradient
#' $\nabla L_2(w, b)$**! To decrease $L_2$ the fastest, we should take a step
#' in the *opposite* direction, $-\nabla L_2(w, b)$.
#'
#' Let's now generalize the gradient descent algorithm that we wrote previously
#' to work our 2-dimensional function.

#' #### Exercise 16
#'
#' Extend your 1D implementation of `gradient_descent` so that it will minimize the function $L_2$.
#'
#' Requirements:
#'
#' * Your new method for `gradient_descent` will take four input arguments: the function $f$ for which you seek the minimum, the range of values for $f$'s first input argument that you will consider, the range of values for $f$'s second input argument that you will consider, and a tolerance that will specify the maximum allowable step size, $\sum_i \eta \frac{\partial f}{\partial p_i}$
#'
#' * Use $\eta = .01$. For example, for a function $f(w, b)$, update $w$ such that $w_{n+1} = w_n - 0.01 * \frac{\partial f}{\partial w_n}$
#'
#' * Seed `gradient_descent` with the starting coordinates [-2.0, -2.0], i.e. $w_0 = -2.0$ and $b_0 = -2.0$.
#'
#' * Return all steps (their coordinates) taken during gradient descent and the values of the loss function at these coordinates.
#'
#' Once you have done this, execute
#'
#' ```julia
#' gradient_descent(L2, -2:0.02:2, -2:0.02:2, .001)
#' ```
#'
#' How many steps were taken by gradient descent?
#'
#' Hint: Do not count your starting coordinates `[-2.0, -2.0]` as a step.

function gradient_descent(fn, wrange, brange; tol=0.001, eta=0.01)
    w = wrange[1]
    b = brange[1]

    w_steps = []
    b_steps = []
    f_steps = []

    while true
        (gfw,gfb) = FiniteDiff.finite_difference_gradient(fn, [w,b])
        w = w - eta * gfw
        b = b - eta * gfb
        f = fn(w,b)

        push!(w_steps, w)
        push!(b_steps, b)
        push!(f_steps, f)

        if abs(eta*gfw + eta*gfb) < tol
            return (w_steps,b_steps,f_steps)
        end

    end

    #return (w_steps, b_steps, f_steps)
end

xs = [2, -3, -1, 1]
ys = [0.8, 0.3, 0.4, 0.4]

sigma(x) = 1/(1 + exp(-x))
g(x,w,b) = sigma(w*x) + b

L2(w,b) = sum(abs2, ys .- g.(xs,w,b))
L2(xb) = L2(xb...)
wrange = -2.0:0.02:2.0
brange = -2.0:0.02:2.0
(w,b,fv) = gradient_descent(L2, wrange, brange)

#' #### Exercise 17
#'
#' Use the `surface` and `scatter!` commands to illustrate the path taken by
#' `gradient_descent` from [-2.0, -2.0] to wherever the algorithm terminates.
#'
#' Where do the scattered points representing the steps of gradient descent
#' appear the most dense?
#'
#' A) Near the starting point at [-2.0, -2.0]

#' B) Near the point [-1.8, 0] where $C_2$ appears nearly flat

#' C) Near the minimum of $C_2$

#' #### Solution
#'
#' B) The following code will plot the function `C2` with an overlay of the
#' steps taken in gradient descent
#'
#' You can change the frequency with which steps are plotted by changing the
#' iterator in the `for` loop from `1:length(values)` to
#' `1:n:length(values)` for varying values of `n`. This will allow you to
#' see more clearly where steps are densely or sparsely packed.

fig = surface(-2:0.02:2, -2:0.02:2, L2, alpha=0.6, xlabel = "w",
             ylabel = "b", zlabel = "C2(w, b)", size=(800,800))
for i in 1:10:length(fv)
   scatter!(fig, [w[i]], [b[i]], [fv[i]], markersize=1)
end
fig


### A Pluto.jl notebook ###
# v0.15.1

using Markdown
using InteractiveUtils

# ╔═╡ 4cb24885-2f27-400e-a20d-b8152f4c59d3
using LinearAlgebra

# ╔═╡ 5e4375cd-a2a4-4027-88a0-a7ae49fbde1c
# we write a function that returns a root of func in the
# interval [low, high] with relative error tol, assuming such a root exists
function binarySearch(func, low, high, tol)

	# we repeat until the tolorance condition is met
	while abs(high - low)/2 > tol * abs(high + low)/2

		# we define mid as the midpoint of the interval [low, high]
		mid = (low + high)/2

		# we check if the value of func at high and mid have the same sign
		if func(high) * func(mid) > 0

			# and reassign [low, high] accordingly
		    high = mid
		else
		    low = mid
		end
	end

	# we return the avarage of low and high as the approximate root
	return (low + high)/2;
end

# ╔═╡ 4533a6ef-85b5-44e2-a381-f60182886903
# we write an improved version of binarySearch that evaluates func fewer times
# and checks if there is definitely a root in the initial interval
function binarySearchV2(func, low, high, tol)

  # we now also store the values of func at low and high
  f_low = func(low)
  f_high = func(high)

  # we check if the initial interval definitely contains a root
  if f_low * f_high > 0
      error("func(low) and func(high) must have different sign")
  end

  # the while loop is similar to before
  while abs(high - low)/2 > tol * abs(low + high)/2
    mid = (low + high)/2

    # we only have one evaluation of func per loop
    f_mid = func(mid)

    if f_high * f_mid > 0
        high = mid
        f_high = f_mid
        # in this case f_low stays the same
    else
        low = mid
        f_low = f_mid
        # in this case f_high stays the same
    end
  end

  return (low + high)/2
end

# ╔═╡ 977bcb22-8089-49f3-931f-be39cbcd950c
f(x) = exp(x)-4x;

# ╔═╡ 25a35c4d-76aa-406b-b6ba-6cbd960c08b9
zeta = 1e-7;

# ╔═╡ d6406dcf-fda4-4fc3-ba27-8f33ac760c80
md"Both implementations find the same approxiamte root in the interval $(0,1)$."

# ╔═╡ 067c0a97-da55-46b9-9476-396b7d60968f
binarySearch(f, 0, 1, zeta)

# ╔═╡ 8a15ea59-e2fc-4534-84dd-52b3c11abeac
binarySearchV2(f, 0, 1, zeta)

# ╔═╡ 031bddca-90a9-43cd-b5d4-e6ccb41f501e
md"Let's test the case where there is no root in the initial interval. Note that binarySearch does not test whether there is a root it gives a bad answer: it is an **unreliable** function."

# ╔═╡ e53a4200-0d48-4d0a-9b60-ac447eff62d4
binarySearch(f, 1, 2, zeta)

# ╔═╡ 57435047-508b-419a-a1b2-8e5cfa4cbfaf
md"BinarySearchV2 on the other hand includes the initial check."

# ╔═╡ a9fd5fbd-8c6e-45b5-9821-ba64175ec3e0
binarySearchV2(f, 1, 2, zeta)

# ╔═╡ aaf5575d-fb60-43b7-858c-46babc9401b1
md"Let's try it for a more complicated function that involves large matrices. We construct a $n\times n$ matrix with random numbers."

# ╔═╡ ebe48c7f-bae4-4262-b792-0581a0757e0d
n = 4000;

# ╔═╡ 5827d97a-5cf3-4578-90ba-11d07afb84d6
A = rand(n,n);

# ╔═╡ ba659a53-9947-4e2d-b615-c3dcd71b3c18
md"Make a symmetric matrix (for real matrices, M' is the transpose of M)."

# ╔═╡ f997e7f0-26d2-4f90-939b-4776a29d5f66
B = A + A';

# ╔═╡ 25142ed0-b0d2-440a-874b-e49fb7fd198c
g(x) = eigmax(exp.(x*B))-2n;

# ╔═╡ 8dfb5f2b-95ef-4701-b90e-2727aa2b2a32
md"""
This function computes a matrix with elements
$A_{ij} = exp(x B_{ij} )$, computes the largest eigenvalue $\lambda$ of $A$ and returns $\lambda - 2n$.
"""

# ╔═╡ f0fdc7d6-9aa3-4a6c-b2e0-f45706d4ae30
g(0), g(1)

# ╔═╡ fe08f690-93a7-46bb-a110-2cb917d12991
md"""
Since $g$ is continuous and $sign(g(0)) \neq sign(g(1))$, it must have a root in the interval $(0,1)$.
"""

# ╔═╡ 0ff24d6c-9032-4743-9400-f90a70b0bc11
binarySearchV2(g, 0, 1, zeta)

# ╔═╡ 00000000-0000-0000-0000-000000000001
PLUTO_PROJECT_TOML_CONTENTS = """
[deps]
LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
"""

# ╔═╡ 00000000-0000-0000-0000-000000000002
PLUTO_MANIFEST_TOML_CONTENTS = """
# This file is machine-generated - editing it directly is not advised

[[Libdl]]
uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"

[[LinearAlgebra]]
deps = ["Libdl"]
uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
"""

# ╔═╡ Cell order:
# ╠═5e4375cd-a2a4-4027-88a0-a7ae49fbde1c
# ╠═4533a6ef-85b5-44e2-a381-f60182886903
# ╠═977bcb22-8089-49f3-931f-be39cbcd950c
# ╠═25a35c4d-76aa-406b-b6ba-6cbd960c08b9
# ╟─d6406dcf-fda4-4fc3-ba27-8f33ac760c80
# ╠═067c0a97-da55-46b9-9476-396b7d60968f
# ╠═8a15ea59-e2fc-4534-84dd-52b3c11abeac
# ╟─031bddca-90a9-43cd-b5d4-e6ccb41f501e
# ╠═e53a4200-0d48-4d0a-9b60-ac447eff62d4
# ╟─57435047-508b-419a-a1b2-8e5cfa4cbfaf
# ╠═a9fd5fbd-8c6e-45b5-9821-ba64175ec3e0
# ╟─aaf5575d-fb60-43b7-858c-46babc9401b1
# ╠═ebe48c7f-bae4-4262-b792-0581a0757e0d
# ╠═5827d97a-5cf3-4578-90ba-11d07afb84d6
# ╟─ba659a53-9947-4e2d-b615-c3dcd71b3c18
# ╠═f997e7f0-26d2-4f90-939b-4776a29d5f66
# ╠═4cb24885-2f27-400e-a20d-b8152f4c59d3
# ╠═25142ed0-b0d2-440a-874b-e49fb7fd198c
# ╟─8dfb5f2b-95ef-4701-b90e-2727aa2b2a32
# ╠═f0fdc7d6-9aa3-4a6c-b2e0-f45706d4ae30
# ╟─fe08f690-93a7-46bb-a110-2cb917d12991
# ╠═0ff24d6c-9032-4743-9400-f90a70b0bc11
# ╟─00000000-0000-0000-0000-000000000001
# ╟─00000000-0000-0000-0000-000000000002


# Examples presented in class - Lecture 1
# Example 4 - Classification

using JuMP, Ipopt

# Generating random multivariate data.
data11 = rand(Normal(8, 4), 100)
data12 = rand(Normal(12, 2), 100)
data1 = [data11 data12]

data01 = rand(Normal(2, 6), 100)
data02 = rand(Normal(5, 2), 100)
data0 = [data01 data02]

scatter(data1[:,1], data1[:,2], label="Pos. obs.")
scatter!(data0[:,1], data0[:,2], label="Neg. obs.")

function classify(data1, data0, δ)
    m = Model(IpoptOptimizer())

    @variable(m, u[1:100] >= 0)
    @variable(m, v[1:100] >= 0)
    @variable(m, a[1:2])
    @variable(m, b)

    @constraint(m, error1[i=1:100], sum(a[j]*data1[i,j] for j = 1:2) - b - u[i] <= -1)
    @constraint(m, error0[i=1:100], sum(a[j]*data0[i,j] for j = 1:2) - b + v[i] >= 1)

# This parameter trades off accuracy and robustness for the classifier
    @objective(m, Min, sum(u[i] for i=1:100) + sum(v[i] for i=1:100)
        + δ*(sum(a[j]^2 for j=1:2)))

    solve(m);

    return getvalue(a), getvalue(b), getvalue(u), getvalue(v)
end

a, b, u1, v1 = classify(data1, data0, 0.01);

# Calculating the points on plane defined by a and b
x1 = linspace(-10,20,100);
x2 = (b - a[1]*x1)/ a[2];
# boundaries of the slab
x2u = (b + 1 - a[1]*x1)/ a[2];
x2d = (b - 1 - a[1]*x1)/ a[2];
plot!(x1, [x2u x2 x2d],
    xlabel = "x_1",
    ylabel = "x_2",
    label= [ "", "Classifier", ""] ,
    color=[:green :green :green],
    linestyle=[ :dot :solid :dot],
    )

# another classifier, with δ=50
a, b, u2, v2 = classify(data1, data0, 100);
x2 = (b - a[1]*x1)/ a[2];
x2u = (b + 1 - a[1]*x1)/ a[2];
x2d = (b - 1 - a[1]*x1)/ a[2];
plot!(x1, [x2u x2 x2d],
    xlabel = "x_1",
    ylabel = "x_2",
    legend = false,
    color=[:purple :purple :purple],
    linestyle=[:dot :solid :dot],
    )


"""
    LogUniform(a,b)

A positive random variable `X` is log-uniformly with parameters `a` and `b` if the logarithm of `X` is `Uniform(log(a), log(b))`.
The *log uniform* distribution is also known as *reciprocal distribution*.
```julia
LogUniform(1,10)
```
External links

* [Log uniform distribution on Wikipedia](https://en.wikipedia.org/wiki/Reciprocal_distribution)
"""
struct LogUniform{T<:Real} <: ContinuousUnivariateDistribution
    a::T
    b::T
    LogUniform{T}(a::T, b::T) where {T <: Real} = new{T}(a, b)
end

function LogUniform(a::T, b::T; check_args::Bool=true) where {T <: Real}
    check_args && @check_args(LogUniform, 0 < a < b)
    LogUniform{T}(a, b)
end

LogUniform(a::Real, b::Real; check_args::Bool=true) = LogUniform(promote(a, b)...; check_args=check_args)

convert(::Type{LogUniform{T}}, d::LogUniform) where {T<:Real} = LogUniform(T(d.a), T(d.b); check_args=false)
Base.minimum(d::LogUniform) = d.a
Base.maximum(d::LogUniform) = d.b

#### Parameters
params(d::LogUniform) = (d.a, d.b)
partype(::LogUniform{T}) where {T<:Real} = T

#### Statistics

function mean(d::LogUniform)
    a, b = params(d)
    (b - a) / log(b/a)
end
function var(d::LogUniform)
    a, b = params(d)
    log_ba = log(b/a)
    (b^2 - a^2) / (2*log_ba) - ((b-a)/ log_ba)^2
end
mode(d::LogUniform)   = d.a
modes(d::LogUniform)  = partype(d)[]

function entropy(d::LogUniform)
    a,b = params(d)
    log(a * b) / 2 + log(log(b / a))
end
#### Evaluation
function pdf(d::LogUniform, x::Real)
    x1, a, b = promote(x, params(d)...) # ensure e.g. pdf(LogUniform(1,2), 1f0)::Float32
    res = inv(x1 * log(b / a))
    return insupport(d, x1) ? res : zero(res)
end
function cdf(d::LogUniform, x::Real)
    x1, a, b = promote(x, params(d)...) # ensure e.g. cdf(LogUniform(1,2), 1f0)::Float32
    x1 = clamp(x1, a, b)
    return log(x1 / a) / log(b / a)
end
logpdf(d::LogUniform, x::Real) = log(pdf(d,x))

function quantile(d::LogUniform, p::Real)
    p1,a,b = promote(p, params(d)...) # ensure e.g. quantile(LogUniform(1,2), 1f0)::Float32
    exp(p1 * log(b/a)) * a
end

function kldivergence(p::LogUniform, q::LogUniform)
    ap, bp, aq, bq = promote(params(p)..., params(q)...)
    finite = aq <= ap < bp <= bq
    res = log(log(bq / aq) / log(bp / ap))
    return finite ? res : oftype(res, Inf)
end


using Plots
using Random
using Pkg; Pkg.add("LaTeXStrings")
using LaTeXStrings

# Set seed
Random.seed!(123)

# Generate MC points
n = 2000
points_x = rand(n)
points_y = rand(n)

# Label points in circle
labels = [x^2 + y^2 <= 1 for (x,y) in zip(points_x, points_y)]

# Estaimte of pi
π_estimate = 4*sum(labels)/n

# Create plot of MC points
plot(points_x, points_y, seriestype = :scatter,
     group = .!labels,
     title = L"\textrm{Monte Carlo } \pi \textrm{ estimation}",
     legend = false,
     bg = RGB(247/255, 236/255, 226/255))

# Add semi-circle quadrant plot
circle_x = 0:0.01:1
circle_y = sqrt.(1 .- circle_x.^2)
plot!(circle_x, circle_y, seriestype= :line, legend = false, lc = :yellow)

# Generate plots comparing π estimate across multiple n
n_values = [5000, 10000, 100000, 1e6, 2e6]
π_estimates_error = zeros(5)

for i in 1:5
    n_value = Int64(n_values[i])
    points_x = rand(n_value)
    points_y = rand(n_value)
    estimate = 4*sum([x^2 + y^2 <= 1 for (x,y) in zip(points_x, points_y)])/
        n_value
    π_estimates_error[i] = 100*abs(π - estimate)/π
end

println(π_estimates_error)

# savefig("../images/pi-monte-carlo.svg")

# Plot area under circle curve
plot(circle_x, circle_y, legend = false,
    fillrange = [zeros(length(circle_x)), circle_y],
    alpha = .5,
    lc = :yellow)


using NumericalMethodsforEngineers
using Test

@sym begin
  ClearAll(xi, yi, Y, Ydot, Ydotdot, ytilde1, ytilde2, R)
  xi = [0, 1//3, 2//3, 1]
  yi = [0, a, b, 1]
  Y(x_) := LagrangePolynomial(xi, yi)
  #
  # Can be re-formulated as Y(x) = F(x) + C1(a) * Ψ1(x) + C2(a) * Ψ2(x)
  #
  # F(x_) := (1/2)*(9x^3 - 9x^2 + 2x)
  # C1(a_) := (27/2)*(a-b)
  # Ψ1(x_) := x^3 - x^2
  # C2(a_) := -(9/2)*(2a-b)
  # Ψ2(x_) := x^2 - x
  #
  # Y(x_) := F(x) + C1(a)*Ψ1(x) + C2(a)*Ψ2(x)
  #
  R(x_) = Simplify(D(Y(x), x, 2) - 3*x - 4*Y(x))
  sol = Solve([R(xi[2]), R(xi[3])], [a, b])
  ytilde(x_) = Y(x) ./ Flatten(sol)
  SetJ(r, ToString(Simplify(R(x))))
  SetJ(s, "[$(sol[1][1][2]), $(sol[1][2][2])]")
  SetJ(t, ToString(Simplify(ytilde(x))))
end

println("\n\nExample 7.15b: y'' = 3x + 4y, y(0)=0, y(1)=1")
println("by 2-point collocation Weighted Residual Method")
@sym Println("\nY(x): ", Y(x), "\n")
@sym Println("R(x) = ", R(x), "\n")
println("(a, b) = $(s)\n")
@sym Println("ytilde_2pt_collocation(x) = ", Simplify(ytilde(x)), "\n")
println()

@eval rf_2pt_collocation_2(x, a, b) = $(Meta.parse(r))
@eval ytilde_2pt_collocation(x) = $(Meta.parse(t))
@eval (a, b) = $(Meta.parse(s))

@test t == "(1/806)*x*(239 - 252x + 819*x^2)"
@test rf_2pt_collocation_2(1//3, a, b) <= eps()
@test rf_2pt_collocation_2(2//3, a, b) <= eps()


# #  Portfolio Optimization
#
# In this problem, we will find the portfolio allocation that minimizes risk while achieving a given expected return $R_\text{target}$.
#
# Suppose that we know the mean returns $\mu \in \mathbf{R}^n$ and the covariance $\Sigma \in \mathbf{R}^{n \times n}$ of the $n$ assets. We would like to find a portfolio allocation $w \in \mathbf{R}^n$, $\sum_i w_i = 1$, minimizing the *risk* of the portfolio, which we measure as the variance $w^T \Sigma w$ of the portfolio. The requirement that the portfolio allocation achieve the target expected return can be expressed as $w^T \mu >= R_\text{target}$. We suppose further that our portfolio allocation must comply with some lower and upper bounds on the allocation, $w_\text{lower} \leq w \leq w_\text{upper}$.
#
# This problem can be written as
#
# $$
# \begin{array}{ll}
#     \text{minimize}   & w^T \Sigma w \\
#     \text{subject to} & w^T \mu >= R_\text{target} \\
#                       & \sum_i w_i = 1 \\
#                       & w_\text{lower} \leq w \leq w_\text{upper}
# \end{array}
# $$
#
# where $w \in \mathbf{R}^n$ is our optimization variable.

using Convex, SCS

## generate problem data
μ = [11.5; 9.5; 6]/100          #expected returns
Σ  = [166  34  58;              #covariance matrix
       34  64   4;
       58   4 100]/100^2

n = length(μ)                   #number of assets

R_target = 0.1
w_lower = 0
w_upper = 0.5;

# If you want to try the optimization with more assets, uncomment and run the next cell. It creates a vector or average returns and a variance-covariance matrix that have scales similar to the numbers above.

#=
using Random
Random.seed!(123)

n = 15                                      #number of assets, CHANGE IT?

μ = (6 .+ (11.5-6)*rand(n))/100             #mean
A = randn(n,n)
Σ = (A * A' + diagm(0=>rand(n)))/500;       #covariance matrix
=#

#-

w    = Variable(n)
ret  = dot(w,μ)
risk = quadform(w,Σ)

p = minimize( risk,
              ret >= R_target,
              sum(w) == 1,
              w_lower <= w,
              w <= w_upper )

solve!(p, SCS.Optimizer())     #use SCS.Optimizer(verbose = false) to suppress printing

#-

# Optimal portfolio weights:
w.value

#-

sum(w.value)


# # [Implicit Nonlinear Dynamics : Autoregulation](@id autoregulation)
#
# The following is another example on how to use the [`ImplicitOptimizer`](@ref) describing a biological autoregulation process using
# two coupled implicit equations.

using DataDrivenDiffEq
using ModelingToolkit
using OrdinaryDiffEq
using LinearAlgebra
#md using Plots
#md gr()

@parameters begin
    t
    α = 1.0
    β = 1.3
    γ = 2.0
    δ = 0.5
end

@variables begin
    x[1:2](t) = [20.0; 12.0]
end

x = collect(x)
D = Differential(t)

eqs = [
    D(x[1]) ~ α/(1+x[2])-β*x[1];
    D(x[2]) ~ γ/(1+x[1])-δ*x[2];
]

sys = ODESystem(eqs, t, x, [α, β, γ, δ], name = :Autoregulation)

x0 = [x[1] => 20.0; x[2] => 12.0]

tspan = (0.0, 5.0)

de_problem = ODEProblem(sys, x0, tspan)
de_solution = solve(de_problem, Tsit5(), saveat = 0.005);
#md plot(de_solution)

# As always, we start by defining a [DataDrivenProblem](@ref problem) and a sufficient basis for sparse regression.

dd_prob = ContinuousDataDrivenProblem(de_solution)

eqs = [
    polynomial_basis(x, 4); D.(x); x .* D(x[1]); x .* D(x[2])
    ]

basis = Basis(eqs, x, independent_variable = t, implicits = D.(x))

#md plot(dd_prob)

# Next to varying over different sparsity penalties, we want also to batch our data using the [DataSampler](@ref).
# We define a train-test split of 80-20 for our data and batch the resulting training data into 10 minibatches, allowing
# shuffled and repeated values. Our goal is to find the model with the minimal error.

sampler = DataSampler(
    Split(ratio = 0.8), Batcher(n = 10, shuffle = true, repeated = true, batchsize_min = 30)
)

res = solve(dd_prob, basis, ImplicitOptimizer(STLSQ(1e-1:1e-1:9e-1)), by = :min, sampler = sampler, digits = 1)
print(res) #hide
print(result(res)) #hide

# We have recovered the correct equations of motion!
# Another visual check using the problem and the result yields

system = result(res)
@named ode = ODESystem(equations(system), t, x, parameters(system));
ode_prob = ODEProblem(ode, x0, tspan, parameter_map(res));

prediction = solve(ode_prob, Tsit5(), saveat = 0.2);
#md plot(de_solution, label = ["Groundtruth" nothing])
#md scatter!(prediction, label = ["Prediction" nothing])

#md # ## [Copy-Pasteable Code](@id autoregulation_copy_paste)
#md #
#md # ```julia
#md # @__CODE__
#md # ```

for r_ in [res] #src
    @test all(l2error(r_) .< 0.5) #src
    @test all(aic(r_) .> 1e3) #src
    @test all(determination(r_) .>= 0.9) #src
end #src


module RandomNumbers
# Note:
# Unless mentioned otherwise, the single variable continuous functions return
# Float64 and the multiple variable continuous functions return Array{Float64, 1}.
# Similarly, the single variable discrete functions return Int64 and the multiple
# variable discrete functions return Array{Int64, 1}.
export rn_unif, rn_exp, rn_standard_cauchy, rn_standard_normal, rn_normal,
	   rn_chi_sq, rn_bernoulli, rn_binom, rn_geo, rn_gamma, rn_beta

# Generate a single uniform random variable between a and b.
function rn_unif(a, b)
	return a + rand() * (b - a)
end

# Generate n random uniform variables between a and b.
function rn_unif(a, b, n::Int64)
	return [rn_unif(a,b) for x in 1:n]
end

# Generate a single exponential random variable with rate parameter theta.
function rn_exp(theta)
	# Don't allow negative theta values.
	if theta <= 0.0
		throw(ArgumentError("Theta cannot be less than zero."))
	end

	return -log(rn_unif(0, 1)) / theta
end

# Generate n exponential random variables with rate parameter theta.
function rn_exp(theta, n::Int64)
	return [rn_exp(theta) for x in 1:n]
end

# Generate a single standard Cauchy random variable.
function rn_standard_cauchy()
	return tan(pi * (rn_unif(0, 1) - .5))
end

# Generate n standard Cauchy random variables.
function rn_standard_cauchy(n::Int64)
	return [rn_standard_cauchy() for x in 1:n]
end

# Generate a single standard normal random variable.
function rn_standard_normal()
	u1 = rand()
	u2 = rand()

	return sqrt(-2 * log(u1)) * cos(2 * pi * u2)
end

# Generate n standard normal random variables.
function rn_standard_normal(n::Int64)
	return [rn_standard_normal() for x in 1:n]
end

# Generate a single normal random variable with mean mu and
# standard deviation sigma.
function rn_normal(mu, sigma)
	# Don't allow negative sigma.
	if sigma <= 0.0
		throw(ArgumentError("Standard deviation cannot be negative."))
	end

	return (sigma * rn_standard_normal()) + mu
end

# Generate n normal random variables with mean mu and standard deviation sigma.
function rn_normal(mu, sigma, n::Int64)
	return [rn_normal(mu, sigma) for x in 1:n]
end

# Generate a single chi-square random normal variable with k degrees of freedom.
# Is specifying Integer in argument necessary?
function rn_chi_sq(k::Integer)
	# Only accept positive natural numbers as input.
	if k < 1
		throw(ArgumentError("Degrees of freedom must be at least 1."))
	end

	# Generate k squared random normal variables.
	normals = rn_standard_normal(k).^2

	# Return the sum of the squared random normal variables.
	return sum(normals)
end

# Generate n chi-square random normal variables with k degrees of freedom.
function rn_chi_sq(k::Integer, n::Int64)
	return [rn_chi_sq(k) for x in 1:n]
end

# Generate a single Bernoulli random variable with success probability
# p as an Int64.
function rn_bernoulli(p)
	# Only allow success probabilities in between 0 and 1.
	if !(0 < p < 1)
		throw(ArgumentError("Success probability must be between 0 and 1."))
	end

	# Generate a single uniform random variable.
	unif = rn_unif(0, 1)

	# Return whether the random variable is below p.
	return unif < p ? 1 : 0
end

# Generate n Bernoulli random variables with success probability p as
# an Array{Int64, 1}.
function rn_bernoulli(p, n::Int64)
	return [rn_bernoulli(p) for x in 1:n]
end

# Generate a single binomial random variable with success probability p.
# Returns Int64.
function rn_binom(trials::Int64, p)
	# Only allow success probabilities in between 0 and 1.
	if !(0 < p < 1)
		throw(ArgumentError("Success probability must be between 0 and 1."))
	end

	# Only allow positive numbers of trials.
	if trials < 1
		throw(ArgumentError("Number of trials must be at least 1."))
	end

	# Generate Bernoulli random variable for each trial.
	berns = rn_bernoulli(p, trials)

	# Return the sum of successful trials.
	return sum(berns)
end

# Generate n binomial random variables with success probability p as an
# Array{Int64, 1}.
function rn_binom(trials, p, n::Int64)
	return [rn_binom(trials, p) for x in 1:n]
end

# Generate a single geometric random variable with success probability p.
function rn_geo(p)
	# Only allow success probabilities in between 0 and 1.
	if !(0 < p < 1)
		throw(ArgumentError("Success probability must be between 0 and 1."))
	end

	lambda = -log(1 - p)

	exp = rn_exp(lambda)

	return ifloor(exp) + 1
end

# Generate n geometric random variables with success probability p.
function rn_geo(p, n::Int64)
	return [rn_geo(p) for x in 1:n]
end

# Generate a single gamma random variable with shape parameter alpha and rate
# parameter beta. This function implements Marsaglia and Tsang's method for
# generating gamma random variables.
function rn_gamma(alpha, beta)
	if alpha <= 0 || beta <= 0
		throw(ArgumentError("Alpha and beta must be greater than zero."))
	end

	if (alpha >= 1)
		# Set two constants used in the algorithm.
		d = alpha - (1.0 / 3.0)
		c = 1.0 / sqrt(9.0 * d)

		# We need to loop until we generate appropriate random numbers
		while true
			# Generate Z~N(0,1).
			z = rn_standard_normal()

			# We need a z that will never result in a negative v.
			if z > -1.0 / c
				v = (1.0 + c * z)^3

				# Generate U~U(0,1).
				u = rn_unif(0, 1)

				# Compute the upper bound for log(u).
				u_bound = (.5 * z^2) + d - (d * v) + (d * log(v))

				# If this condition holds we've found a suitable result.
				if log(u) < u_bound
					return (d * v) / beta
				end
			end
		end
	else
		# When 0 < alpha < 1 we can return a transformed answer.
		x = rn_gamma(alpha + 1, beta)
		u = rn_unif(0, 1)
		return x * u ^ (1 / alpha)
	end
end

# Generate n gamma random variables with shape parameter alpha and rate
# parameter beta.
function rn_gamma(alpha, beta, n::Int64)
	return [rn_gamma(alpha, beta) for x in 1:n]
end

# Generate a beta random variable with shape parameters alpha and beta.
# Note that this does not use the rejection method.
function rn_beta(alpha, beta)
	if alpha <= 0 || beta <= 0
		throw(ArgumentError("Alpha and beta must be greater than zero."))
	end

	# Generate two independent gamma variables
	x = rn_gamma(alpha, 1)
	y = rn_gamma(beta, 1)

	return x / (x + y)
end

function rn_beta(alpha, beta, n::Int64)
	return [rn_beta(alpha, beta) for x in 1:n]
end

end


# # Consumption Savings Problem
# In this case study, a household endowed with ``B_0`` dollars of wealth
# must decide how much to consume and save
# to maximize its utility over its finite lifecycle.

# ## Formulation

# The corresponding dynamic optimization problem is expressed:
# ```math
# \begin{aligned}
# 	&V(B_0,0) = &&\underset{c(t), B(t)}{\text{max}} \int_{t = 0}^{t=T} e^{-\rho t} u(c(t)) dt \\
# 	&\text{s.t.} \\
# 	&&& \frac{dB}{dt} = r \times B(t) - c(t), && t \in [0,T] \\
#   &&& B(0) = B_0 \\
#   &&& B(T) = 0
# \end{aligned}
# ```
# where
# the household lives during time ``t \in [0,T]``,
# the state variable ``B(t)`` is the household's stock of wealth at time ``t``,
# the choice variable ``c(t)`` is the household's consumption at time ``t``,
# ``r`` is the interest rate,
# ``B_0`` is the household's wealth endowment (initial condition),
# ``B(T) = 0`` is the terminal condition,
# and ``T`` is the time
# horizon.

# ## Model Definition

# Let's implement this in `InfiniteOpt` and first import the packages we need:
using InfiniteOpt, Ipopt

# We set the preference and constraint parameters:
ρ = 0.025  # discount rate
k = 100.0  # utility bliss point
T = 10.0   # life horizon
r = 0.05   # interest rate
B0 = 100.0 # endowment
u(c; k=k) = -(c - k)^2       # utility function
discount(t; ρ=ρ) = exp(-ρ*t) # discount function
BC(B, c; r=r) = r*B - c      # budget constraint

# We set the hyperparameters:
opt = Ipopt.Optimizer   # desired solver
ns = 1_000;             # number of points in the time grid

# We initialize the infinite model and choose the Ipopt solver:
m = InfiniteModel(opt)

# Let's specify our infinite parameter which is time ``t \in [0, T]``:
@infinite_parameter(m, t in [0, T], num_supports = ns)

# Now let's specify the variables:
@variable(m, B, Infinite(t)) ## state variables
@variable(m, c, Infinite(t)) ## control variables

# Specify the objective:
@objective(m, Max, integral(u(c), t, weight_func = discount))

# Set the initial/terminal conditions:
@constraint(m, B == B0, DomainRestrictions(t => 0))
@constraint(m, B == 0, DomainRestrictions(t => T))

# Set the budget constraint:
@constraint(m, c1, deriv(B, t) == BC(B, c; r=r))

# ## Problem Solution

# Optimize the model:
optimize!(m)
termination_status(m)

# Extract the results:
c_opt = value(c)
B_opt = value(B)
ts = supports(t)
opt_obj = objective_value(m) # V(B0, 0)

# Plot the results:
using Plots
ix = 2:(length(ts)-1) # index for plotting
plot(ts[ix],   B_opt[ix], lab = "B: wealth balance")
plot!(ts[ix],  c_opt[ix], lab = "c: consumption")

# That's it, now we have our optimal trajectory!

# This very simple problem has a closed form solution:
λ1 = exp((r)T)
λ2 = exp(-(r-ρ)T)
den = (λ1-λ2)r
Ω1 = (k + (r*B0-k)λ2)/den
Ω2 = (k + (r*B0-k)λ1)/den
c0 = r*B0 + (r)Ω1 + (r-ρ)Ω2
BB(t; k=k,r=r,ρ=ρ,Ω1=Ω1,Ω2=Ω2) = (k/r) - Ω1*exp((r)t) + Ω2*exp(-(r-ρ)t)
cc(t; k=k,r=r,ρ=ρ,c0=c0)       = k + (c0-k)*exp(-(r-ρ)t)

# Compare the solution given by `InfiniteOpt` with the closed form:
plot(legend=:topright);
plot!(ts[ix], c_opt[ix], color = 1, lab = "c: consumption, InfiniteOpt");
plot!(ts[ix], cc, color = 1, linestyle=:dash, lab = "c: consumption, closed form");
plot!(ts[ix], B_opt[ix], color = 4, lab = "B: wealth balance, InfiniteOpt");
plot!(ts[ix], BB, color = 4, linestyle=:dash, lab = "B: wealth balance, closed form")

# Not bad!

# ### Maintenance Tests
# These are here to ensure this example stays up to date.
using Test
@test termination_status(m) == MOI.LOCALLY_SOLVED
@test has_values(m)
@test B_opt isa Vector{<:Real}
@test c_opt isa Vector{<:Real}


"""
# chebyshevfun
普通递归给出第一类切比雪夫函数的表达式
```math
T_n(x)=2xT_{n-1}(x)-T_{n-2}(x)
```
"""
function chebyshevT(n::Int64,x)
    if n<0
        error("chebyshev(n,x), n>=0 is violated")
    elseif n==0
        return 1
    elseif n==1
        return x
    elseif n==2
        return 2*x^2-1
    elseif n==3
        return 4*x^3-3*x
    elseif n==4
        return 8*x^4-8*x^2+1
    elseif n==5
        return 16*x^5-20*x^3+5*x
    elseif n==6
        return 16*x^5-20*x^3+5*x
    else
        return 2*x*chebyshevT(n-1,x)-chebyshevT(n-2,x)
    end
end

"""
# chebyshevfun
普通递归给出第一类切比雪夫函数的表达式
```math
U_n(x)=2xU_{n-1}(x)-U_{n-2}(x)
```
"""
function chebyshevU(n::Int64,x)
    if n<0
        error("chebyshev(n,x), n>=0 is violated")
    elseif n==0
        return 1
    elseif n==1
        return 2*x
    elseif n==2
        return -1 + 4*x^2
    elseif n==3
        return -4*x + 8*x^3
    elseif n==4
        return 1 - 12*x^2 + 16*x^4
    elseif n==5
        return 6*x - 32*x^3 + 32*x^5
    elseif n==6
        return -1 + 24*x^2 - 80*x^4 + 64*x^6
    else
        return 2*x*chebyshevU(n-1,x)-chebyshevU(n-2,x)
    end
end

# """
# # chebyshevexpand
# 切比雪夫展开函数
# 输入一个函数，返回在指定点处切比雪夫展开后的版本
# """
"""
为切比雪夫T而写的宏
"""
macro chebyT(n,x)
    return esc(:(2*$x*chebyshevT($n-1,$x)-chebyshevT($n-2,$x)))
end

"""
# chebyshevTmeta
为切比雪夫展开的多项式
用元编程进行操作
"""
function chebyshevTmeta(n::Int64,x)
    if n==0
        return 1
    elseif n==1
        return x
    elseif n>=2
        return @chebyT(n,x)
    end
end

# """
# 为切比雪夫U而写的宏
# """
# macro chebyU(n,x)
#     return esc(:(2*x*chebyshevT($n-1,$x)-chebyshevT($n-2,$x)))
# end

# """
# # chebyshevTmeta
# 为切比雪夫展开的多项式
# 用元编程进行操作
# """
# function chebyshevUmeta(n::Int64,x)
#     if n==0
#         return 1
#     elseif n==1
#         return 2*x
#     elseif n>=2
#         return @chebyU(n,x)
#     end
# end


"""
    euler_totient(n::Int64)

Compute the positive integers up to a given integer n that are relatively prime to n; also
known as Euler's totient. For more information see: [https://en.wikipedia.org/wiki/Euler%27s_totient_function](https://en.wikipedia.org/wiki/Euler%27s_totient_function)

# Arguments
- `n::Int64`: Number of elements

# Examples
```julia-repl
julia> import ClassicAlgorithmsCollections
julia> ClassicAlgorithmsCollections.euler_totient(20)
[1, 1, 2, 2, 4, 2, 6, 4, 6, 4, 10, 4, 12, 6, 8, 8, 16, 6, 18, 8]
```
"""
function euler_totient(n::Int64)

    result = collect(Int64, 1:n)

    for i in 1:n
        m = i
        p = 2
        while m >= p^2
            if (m % p == 0)
                while (m % p == 0)
                    m = convert(Int64, m / p)
                end
                result[i] -= convert(Int64, result[i] / p)
            end
            p += 1
        end
        if m > 1
            result[i] -= convert(Int64, result[i] / m)
        end
    end
    return result
end


using Revise
using DynamicalSystems
using Basins
using Plots
using BenchmarkTools

# FRACTAL BASIN BOUNDARIES
# Steven W. MCDONALD, Celso GREBOGY Edward OTT and James A. YORKE
# Physica D 17 1985
function chaotic_map(dz,z, p, n)

    xn = z[1]; yn = z[2]
    λ₁=p[1];λ₂=p[2];
    dz[1]=mod(λ₁*xn,1)
    dz[2]=λ₂*yn+cos(2*π*xn)
    return
end

# dummy function to keep the initializator happy
function chaotic_map_J(J,z0, p, n)
   return
end
ds = DiscreteDynamicalSystem(chaotic_map,[1., 0.], [1.1, 1.] , chaotic_map_J)
integ  = integrator(ds)

# This map has two escapes to ±∞
function escape_function(y)
    if y > 10000
        return 1
    elseif y < -10000
        return 2
    end
    return 0
end

function  get_color(integ,x,y)
    reinit!(integ,[x,y])
    while escape_function(integ.u[2]) == 0
        step!(integ)
        integ.t > 2000 && break
    end
    return escape_function(integ.u[2]);
end

xg=range(0.,1.,length=600)
yg=range(-2.,2.,length=600)
integ.p[1] = 3
integ.p[2] = 1.5
basin_t = zeros(length(xg),length(yg))

# Compute escape basin
for (k,x) in enumerate(xg), (n,y) in enumerate(yg)
    basin_t[k,n]=get_color(integ,x,y);
end

@show bd = box_counting_dim(xg, yg, basin_t)

# Estimation using the method of the paper.
# The results do not match exactly. But the paper talks about
# approximate results
N=[]
eps_r =  10 .^range(-8,-1,length=10)

for eps in eps_r
    @show eps
    N2=0
    N1=0

    for k in 1:8192*20

        k1 = rand(1:length(xg))
        k2 = rand(1:length(yg))

        c1 = get_color(integ, xg[k1],yg[k2])
        c2 = get_color(integ, xg[k1]+eps,yg[k2])
        c3 = get_color(integ, xg[k1]-eps, yg[k2])

        if length(unique([c1,Int(c2),Int(c3)]))>1
            N1 += 1
        end
        N2 += 1
    end
    @show N1/N2
    push!(N,N1/N2)
end

using LsqFit
# get exponent
@. model(x, p) = p[1]*x+p[2]
fit = curve_fit(model, vec(log10.(eps_r)), vec(log10.(N)), [2., 2.])
D = coef(fit)
@show 2-D[1]

plot(xg,yg,basin_t',seriestype=:heatmap)


```
Reg(args...)  -->Plot

Estimate Linear, Exponential and Power regression type
``Linear\quad \Rightarrow y=mx+b\\ Exponential\quad \Rightarrow y=b{ e }^{ mx }\\ Power\quad \Rightarrow y=b{ x }^{ m }``

The next table show the list of variables allowed:

|PropertyName|Args|Default|Input|Description
|---|---|---|---|---|
|x|Mandatory|--|Array{Number,1}| x Data|
|y|Mandatory|--|Array{Number,1}| y Data|
|TypeReg|Optional|TypeReg="Linear"|TypeReg= "linear" or "Exponential"  or "Power"|Type of regression|
|pr|Optional|pr=false|pr=Bool|Print the results of regression|
```
function Reg(x,y; TypeReg="Linear", pr=false)

    if TypeReg=="Linear"
        m=(mean(x)*mean(y)-mean(x.*y))/(mean(x)^2-mean(x.^2))
        b=mean(y)-m*mean(x)
        SELine=sum(map((X,Y)->((X*m+b)-Y)^2,x,y))
        SEYmean=sum(map(Y->(Y-mean(y))^2,y))
        R2=1-(SELine/SEYmean)

        pr ? println(" **Linear Regression \n form y = mx+b \n m=$m \n b=$b \n r2=$R2") : :none
        return m,b,R2
    elseif TypeReg=="Exponential"
        m=(mean(x)*mean(log.(y))-mean(x.*log.(y) ) )/(mean(x)^2-mean(x.^2))
        Logb=mean(log.(y))-m*mean(x)
        b=exp(Logb)
        SELine=sum(map((X,Y)->((X*m+Logb)-Y)^2,x,log.(y)))
        SEYmean=sum(map(Y->(Y-mean(log.(y)))^2,log.(y)))
        R2=1-(SELine/SEYmean)

        pr ? println(" **Exponential Regression \n form y = b*exp(mx) \n m=$m \n b=$b \n r2=$R2") : :none
        return m, b,R2
    elseif TypeReg=="Power"
        m=(mean(log.(x))*mean(log.(y))-mean(log.(x).*log.(y) ) )/(mean(log.(x))^2-mean(log.(x).^2))
        Logb=mean(log.(y))-m*mean(log.(x))
        b=exp(Logb)
        SELine=sum(map((X,Y)->((X*m+Logb)-Y)^2,log.(x),log.(y)))
        SEYmean=sum(map(Y->(Y-mean(log.(y)))^2,log.(y)))
        R2=1-(SELine/SEYmean)

        pr ? println(" **Power Regression \n form y = b x^(m) \n m=$m \n b=$b \n r2=$R2") : :none
        return m, b,R2
    end

end


### A Pluto.jl notebook ###
# v0.18.4

using Markdown
using InteractiveUtils

# This Pluto notebook uses @bind for interactivity. When running this notebook outside of Pluto, the following 'mock version' of @bind gives bound variables a default value (instead of an error).
macro bind(def, element)
    quote
        local iv = try Base.loaded_modules[Base.PkgId(Base.UUID("6e696c72-6542-2067-7265-42206c756150"), "AbstractPlutoDingetjes")].Bonds.initial_value catch; b -> missing; end
        local el = $(esc(element))
        global $(esc(def)) = Core.applicable(Base.get, el) ? Base.get(el) : iv(el)
        el
    end
end

# ╔═╡ cf07621a-9a65-11ec-1fc0-51fbadbb086f
using Pkg; Pkg.activate()

# ╔═╡ 345a90f7-f8ac-43b2-b5bd-0282d9447556
begin
	using DifferentialEquations
	using ModelingToolkit
	using DynamicalSystems
	using ForwardDiff
	using LinearAlgebra
	using Graphs
	using Plots
	using PlutoUI
	using LaTeXStrings
end

# ╔═╡ eaf9dda2-d898-4771-9f86-ca4be174712c
md"""
# Dynamical Systems in Julia
In this tutorial, we'll looking at specifying, simulating and analyzing dynamical systems in Julia. This is one area where Julia especially shines, thanks in no small part to the `DifferentialEquations` package, which pretty much provides the largest and fastest suite of ODE solvers on the market today.
"""

# ╔═╡ eb6aed5a-0f95-4a76-8a99-b8719984bbe5
md"## Defining systems symbolically"

# ╔═╡ 0e28ede5-0af6-49a7-b95d-7094e320eee4
md"""
We'll start by using ModelingToolkit to define the famous Lorenz system. The process starts by a symbolic description of the problem.
"""

# ╔═╡ 6ccfc87f-40e7-4ee4-baae-ee41b81efe67
begin
	@parameters t σ ρ β
	@variables x(t) y(t) z(t)
	D = Differential(t)

	eqs = [
		D(x) ~ σ*(y-x),
       	D(y) ~ x*(ρ-z)-y,
       	D(z) ~ x*y - β*z
	]

	@named lorenz = ODESystem(eqs)
end

# ╔═╡ 3e4ec586-482f-4c1b-a8e7-4ae88e48fa38
md"""
So far, the `lorenz` is purely symbolic, which has the added benefit of being able to render the equations in LaTeX as above! The next step before simulating it is defining values for our initial condition and parameters.
"""

# ╔═╡ ce0b65b5-e57a-4def-acb6-892c49f182d9
begin
	u0 = [
		x => 0.0,
      	y => 0.5,
      	z => 0.5
	]

	p  = [
		ρ => 28.0,
      	σ => 10.0,
      	β => 8/3
	]
end

# ╔═╡ 1a7e8273-8069-42a8-ab7b-62f5fe1ade67
md"""
Simulating our system is a two step process. After defining our timespan of integration, we first create an `ODEProblem`, we describes an ODE + initial conditions and parameters, and then pass it to the `solve` function to compute the solution of our problem. The second argument to solve is the solver we want to use (`Tsit5` is a 4-5 order Runge-Kutta scheme which is generally a good default choice for non-stiff problems)
"""

# ╔═╡ 1d77a2be-773c-4edd-9d68-1ed84fb5dadf
begin
	tspan = (0.0,100.0)
	prob = ODEProblem(lorenz,u0,tspan,p,jac=true)
	sol = solve(prob,Tsit5())
end

# ╔═╡ 7c712f73-3931-4cf4-82d8-9d9e2263df14
plot(sol, vars=(x,y,z))

# ╔═╡ 432fa70e-9225-4e76-aec3-c27d31887fb9
md"""
NB. There are also packages for modeling specific classes of dynamical systems, such as `Catalyst` for chemical reaction networks, or `PowerDynamics` for power grids.
"""

# ╔═╡ 60063cc3-e71f-461b-be0a-491b1b566718
md"## Defining Systems by hand"

# ╔═╡ ea2eeac2-5514-4fe8-8921-6235cdb0d1ad
md"""
While defining systems symbolically is cool, this can be tedious for large systems, and sometimes you just want full control over how your ODE function is defined. To that end, we will now define another famous system, the Brusselator by hand and then build a network reaction diffusion system with it.

The Brusselator equations are as follows,
```math
\begin{align*}
\dot{x} &= a + x^2y -bx -x \\
\dot{y} &= bx - x^2y
\end{align*}
```
"""

# ╔═╡ 2bf13604-71c6-474f-90de-5b270c5c0708
md"""
We can construct ODEProblems by providing a function that computes the derivative, just like we do in Matlab. The functions we can use come in two flavours. The first type just takes as input the state `u`, parameters stored in `p` and time `t` (for non autonomous systems), and returns the new state.
"""

# ╔═╡ 743b31cd-6b2a-472c-8a70-a3955a2c26b6
function brusselator(u, p, t)
	a, b = p
	[
		a + u[1]^2 * u[2] - b*u[1] - u[1],
		b*u[1] - u[1]^2 * u[2]
	]
end

# ╔═╡ 03bd6ff6-107f-4b04-9cb8-fe5c28570e58
md"""
However, it is always recommended to use the second form, which also takes as input a preallocated array for the output and modifies it. Doing this is more efficient in terms of memory allocation, which is actually an important source of slowdowns.
"""

# ╔═╡ 08fc4616-d2af-471b-adc0-195bb566334f
function brusselator!(du, u, p, t)
	a, b = p
	du[1] = a + u[1]^2 * u[2] - b*u[1] - u[1]
	du[2] = b*u[1] - u[1]^2 * u[2]
end

# ╔═╡ d0c1e91e-d1df-4a12-a0e2-8513eeee7ba6
begin
	u0_b = [1.0, 1.0]
	p_b = [1.0, 3.0]
end

# ╔═╡ 8623c9b7-5724-4678-9b53-4312389a276d
begin
	prob_b = ODEProblem(brusselator!, u0_b, (0.0, 50.0), p_b)
	sol_b = solve(prob_b, Tsit5())
end

# ╔═╡ da0220ea-99aa-4a0d-8826-f0e461fd503e
plot(sol_b)

# ╔═╡ 60d1ea9b-b935-4e88-9e92-07497f06aeaa
md"""
Now that we have our brusselator function, let's use it on a network to make a reaction diffusion system. Such a system is of the form

```math
\dot{u_i} = f(u_i) + D \sum_{j=1}^N A_{ij}(u_j - u_i),
```
where ``u_i\in\mathbb{R}^m`` is the state of node ``i``, subject to local dynamics ``f:\mathbb{R}^m\rightarrow \mathbb{R}^m``, ``A\in\mathbb{R}^{N\times N}`` is the adjacency matrix of the graph (i.e. ``A_{ij}=1`` iff there is a link between nodes ``i`` and ``j``), and ``D`` is a diagonal matrix of diffusion coefficients.

We can rewrite this system in vector form (glossing over some details) as

```math
\dot{\mathbf{u}} = F(\mathbf{u}) + \mathbf{D} L \mathbf{u},
```
where ``L`` is the *Laplacian matrix* of the graph.

Reaction-Diffusion systems can take more general forms, but we'll stick to this one for today.
"""

# ╔═╡ efbb46be-c442-4ab3-85a0-b10cbc3a2a9e
md"""
Now, we need to create a function for our reaction-diffusion system.
"""

# ╔═╡ 8b2ec98a-b37f-4bd0-9fc8-9f3dcf9550c9
function brusselator_rd(u, p, t)
	a, b, L, D = p
	du = similar(u) # creates an array of same shape and element type as u
	for i in 1:size(du, 1)
		du[i,:] .= brusselator(u[i,:], [a, b], t)
	end
	du[:,1] .-= D[1] .* (L*u[:,1])
	du[:,2] .-= D[2] .* (L*u[:,2])
	return du
end

# ╔═╡ 25c22a1f-6ae4-4b72-8e4c-de844e2699d1
function brusselator_rd!(du, u, p, t)
	# NB. the commented out parts should be equivalent to what is used, but for some reason it is not
	a, b, L, D = p
	du .= 0.0
	for i in 1:size(du, 1)
		#brusselator!(du[i,:], u[i,:], [a, b], t)
		du[i,:] .+= brusselator(u[i,:], [a, b], t)
	end
	#mul!(du[:,1], L, u[:,1], -D[1], 0.0) # du[:,1] .+= D[1]*L*u[:,1]
	#mul!(du[:,2], L, u[:,2], -D[2], 0.0)
	du[:,1] .-= D[1] .* (L*u[:,1])
	du[:,2] .-= D[2] .* (L*u[:,2])
end

# ╔═╡ 582b74c7-aaf6-4a8e-901f-489199ccc0d6
md"""
Now, we need a network to run our system on. To do this, we can use the `Graphs` package.
"""

# ╔═╡ 3e575a45-50d6-4297-84e5-6699e36d00dd
g = erdos_renyi(10, 0.4)

# ╔═╡ ed1de34a-a42e-474d-ba36-411163a343f8
md"""
We can use the function `laplacian_matrix` to get the Laplacian matrix of the graph. The nice thing about this function is that it returns a sparse matrix, which uses optimized methods for matrix multiplications.
"""

# ╔═╡ 947c169c-a900-4728-ad29-5200b52f9599
laplacian_matrix(g)

# ╔═╡ 04bbc594-17c9-4b41-a599-d41e2e541c9f
begin
	u_rd_0 = 5*rand(nv(g), 2)
end

# ╔═╡ 29a0f8c8-5529-496c-99f1-b436d15f2517
md"""
Here's a nice feature of Pluto. It provides allows interactively modifying variables with html elements (sliders, buttons, ...) via the `PlutoUI` package.

Let's use it to make sliders for the parameters of our system.
"""

# ╔═╡ f346a0b3-4d1d-49ad-94ec-cc42ee75a05b
md"""
a: $(@bind a Slider(0.0:0.01:5.0, default=1.0, show_value=true))

b: $(@bind b Slider(0.0:0.01:5.0, default=3.0, show_value=true))

D₁: $(@bind D_1 Slider(0.0:0.01:10.0, default=1.0, show_value=true))

D₂ $(@bind D_2 Slider(0.0:0.01:10.0, default=1.0, show_value=true))
"""

# ╔═╡ 8a1c55e5-1008-44ee-b853-c53c2dbf3bb2
p_rd = [a, b, laplacian_matrix(g), [D_1, D_2]];

# ╔═╡ 2ee789c5-ae38-4e85-95e1-920a580df058
begin
	prob_rd = ODEProblem(brusselator_rd!, u_rd_0, (0.0, 50.0), p_rd)
	sol_rd = solve(prob_rd, Tsit5())
	nothing
end

# ╔═╡ 9d1f3408-e0f5-4093-bb6b-862485e99476
begin
	local p1 = plot(sol_rd, vars=(0,1:nv(g)), label="", ylabel=L"x_i")
	local p2 = plot(sol_rd, vars=(0,(nv(g)+1:2nv(g))), label="", ylabel=L"y_i")
	plot(p1, p2)
end

# ╔═╡ a4d4a26f-ff25-4bf0-b8ab-2db56131ee3e
md"""
### Exercise
Implement your favourite dynamical system and play around with it's parameters.
"""

# ╔═╡ 91f1ce37-0d68-4ed2-9359-dbcfd06f6cdd
md"""
## Intermezzo: using `DynamicalSystems` to study chaotic Systems

Before moving to the last part of this tutorial, let us make a detour to look at the `DynamicalSystems` package, which provides a library of famous dynamical systems, as well as generic tools for ananlyzing them, particularly chaotic systems.
"""

# ╔═╡ 4328e814-82a8-4e7d-8b8a-44a9d5ba17c5
ds = Systems.logistic()

# ╔═╡ 7b11ac9b-a470-4059-bbad-ba6bc443c68b
md"""
Here we load the well known logistic map from the library of dynamical systems, and generate its orbit diagram for varying values of the `r` parameter. The magic function is `orbitdiagram`, which should work for pretty much any discrete map.
"""

# ╔═╡ a5629c28-53bf-4b19-a49f-ab26df1d1b46
begin
	i = 1
	pvalues = 1:0.005:4
    ics = [rand() for m in 1:10]
    n = 2000
    Ttr = 2000
    p_index = 1

    output = orbitdiagram(ds, i, p_index, pvalues; n = n, Ttr = Ttr)
end

# ╔═╡ 74adb98a-1ad5-4181-a3be-ec5fe2565a35
begin
	local L = length(pvalues)
	local x = Vector{Float64}(undef, n*L)
	local y = copy(x)
	for j in 1:L
    	x[(1 + (j-1)*n):j*n] .= pvalues[j]
  		y[(1 + (j-1)*n):j*n] .= output[j]
	end
	scatter(x,y,
		xlabel="r",
		ylabel="x",
		label="",
		markersize=0.8,
		color=:black,
		markeralpha=0.05,
		markerstrokealpha=0.0
	)
end

# ╔═╡ c38e202f-a2d2-4449-8308-df22774b7af1
md"""
NB. While `DynamicalSystems` provides a bunch of cool tools, a lot of them seem to inexplicably break when used outside of the examples. For example, try changing the above code to use values of `r` above 4.
"""

# ╔═╡ d5ab9847-ace4-4a5b-8c81-d2eb8b56c896
md"""
## Analyzing the stability of our system

Reaction-Diffusion systems like the one we just saw are interesting because of their asymptotic behaviour.  Because the laplacian always has a zero eigenvalue associated to an eigenvector whose entries are all the same, a reaction-diffusion system always admits a homogeneous solution where the nodes are all synchronised.
By tuning the diffusion coefficients, it's possible to (de)synchronize the states of the nodes.

The standard tool for analysing this phenomenon is the Master Stability Function [Pecora-Caroll. 1998].

The main idea consists in expressing the system in the basis of eigenvectors of the laplacian matrix. By considering a small perturbation around the synchronized solution, and expressing its linearized equations in the eigenbasis of ``L``, one can obtain a set of decoupled equations involving the laplacian eigenvalues and the jacobian of the local dynamics.

Skipping over the details, we can check that the synchronized solution is linearly stable by checking that the following ODE is stable for every eigenvalue ``\lambda`` of the laplacian matrix

```math
\dot{\xi} = [\mathcal{J}f(u_*) - \lambda D] \xi
```
where ``\mathcal{J}f`` denotes the jacobian of ``f``.

This can be done, for example, by computing the Lyapunov exponent of this equation, thereby yielding a function ``MSF(\lambda)`` called the *Master Stability Function* of the system.
"""

# ╔═╡ 575e3106-704e-4bea-8155-7ee6e6c9da5d
md"First, let's compute the eigenvalues of the laplacian matrix. We don't actually need to compute the eigenvectors."

# ╔═╡ bff34b78-f7ad-452e-9b95-a498f3e377f5
λ = laplacian_spectrum(g)

# ╔═╡ bba5eaa1-16f2-401d-8cf3-f05e9761583d
md"""
Next, let's start by writing a function that takes the local dynamics as input, and generates a function for the MSF ODE.

Notice the use, of `ForwardDiff.jacobian` to automagically compute the Jacobian matrix using Automatic Differentiation (this is different from Finite Differences or Symbolic methods)
"""

# ╔═╡ ddbb84d1-945a-4184-850b-31ee3540e960
function msf_ode_fun(f)
	function msf_ode!(du, u, p, t)
		p_f, D, λ = p
		du[:,1] .= f(u[:,1], p_f, t)
		J = ForwardDiff.jacobian(x -> f(x,p_f,t), u[:,1])
		du[:,2] .= J*u[:,2] - λ*D*u[:,2]
	end
	return msf_ode!
end

# ╔═╡ f0215a78-146d-4009-ae73-8a2f35072db9
md"""
Now, we can move on to computing the MSF proper. To do this, we solve the MSF ODE, then estimating the Lyapunov exponent from the time series.

NB. Recall that the Lyapunov exponent for a dynamical sytem is defined by
```math
\lambda = \lim_{t\rightarrow \infty} \lim_{\xi_0 \rightarrow 0} \frac{1}{t} \ln \frac{\|\xi(t)\|}{\|\xi_0\|},
```
where ``\xi(0) = \xi_0`` is a small perturbation applied to the orbit of interest (here, the periodic solution).
"""

# ╔═╡ 84d86795-e2c4-4c2c-888b-ae999bcda2dd
function master_stability_function(f, u_0, p_f, D, t; δ=0.1, k=50, Δt=0.1)
	msf_ode! = msf_ode_fun(f)
	n = length(u_0)
	u_msf_0 = hcat(u_0, δ*randn(n))
	function msf(λ; u_msf_0=u_msf_0, p_f=p_f, D=D, t=t, Δt=Δt)
		p_msf = [p_f, D, λ]
		prob = ODEProblem(msf_ode!, u_msf_0, (0.0,t), p_msf)
		sol = solve(prob, Tsit5(), saveat=0.0:Δt:t, abstol=1e-10, reltol=1e-10)
		ξ_0 = norm(sol.u[1][:,2])
		l = length(sol.u)
		ξ_t = norm([sol.u[k][:,2] for k in l-k:l])
		return log(ξ_t/ξ_0)/sol.t[end]
	end
	return msf
end

# ╔═╡ 8c592351-4da7-4bed-a99c-595156c1ed8b
begin
	u0_msf = sol_b.u[end]
	p_f = [1.0, 3.0]
	D_msf = Diagonal([0.2,5.0])
end

# ╔═╡ e3411cdb-acf7-40da-9f5e-6852c1370797
msf = master_stability_function(brusselator, u0_msf, p_f, D_msf, 200.0)

# ╔═╡ 5fbfe4aa-f2e5-4845-92e7-9de00063059f
msf(0.0)

# ╔═╡ ffecadd1-95b5-412f-86e4-e4dee733cab1
md"""
Note that this implementation is not super fast, as it takes about a minute to run the next cell. There's lots of room for improvements though, and in the mean time, we still have a generic funcion for computing the MSF.
"""

# ╔═╡ bad93338-72e3-46fc-888f-e507c62cefc6
begin
	local λs = 0.0:0.01:7.0
	plot(λs, msf.(λs),
		label="",
		xlabel="-Re λ"
	)
	#plot!(λs, msf_ds.(λs), label="ds")
	plot!([0.0,7.0],[0.0,0.0], linestyle=:dash, label="", color=:black)
	scatter!(λ, msf.(λ), label="")
end

# ╔═╡ 63c14c9e-808e-464b-b5cc-25921aa34f56
md"""
Below, we examine the trajectory of the MSF equations for λ = 0. Note that this is quite ill-behaved (notice the log scale), so it's no wonder this is tricky to get right.
"""

# ╔═╡ 8f8a9dec-4de2-4fc7-9d87-a3638ec47ee3
begin
	f! = msf_ode_fun(brusselator)
	u0_ = hcat(u0_msf, 0.1*randn(2))
	p_msf = [p_f, D_msf, 0.0]
	prob_msf = ODEProblem(f!, u0_, (0.0,100.0), p_msf)
	sol_msf = solve(prob_msf, Rodas5(), abstol=1e-10, reltol=1e-10)
end

# ╔═╡ e85b9b6d-736d-446a-acb1-b6ed0ccdd907
begin
	local foo = (t,u,v) -> (t,norm([u;v]))
	plot(sol_msf, vars=(foo,0,3,4),
		yscale=:log10,
		label="",
		ylabel=L"\ln \|\xi\|"
	)
end

# ╔═╡ Cell order:
# ╟─eaf9dda2-d898-4771-9f86-ca4be174712c
# ╠═cf07621a-9a65-11ec-1fc0-51fbadbb086f
# ╠═345a90f7-f8ac-43b2-b5bd-0282d9447556
# ╟─eb6aed5a-0f95-4a76-8a99-b8719984bbe5
# ╟─0e28ede5-0af6-49a7-b95d-7094e320eee4
# ╠═6ccfc87f-40e7-4ee4-baae-ee41b81efe67
# ╟─3e4ec586-482f-4c1b-a8e7-4ae88e48fa38
# ╠═ce0b65b5-e57a-4def-acb6-892c49f182d9
# ╟─1a7e8273-8069-42a8-ab7b-62f5fe1ade67
# ╠═1d77a2be-773c-4edd-9d68-1ed84fb5dadf
# ╠═7c712f73-3931-4cf4-82d8-9d9e2263df14
# ╟─432fa70e-9225-4e76-aec3-c27d31887fb9
# ╟─60063cc3-e71f-461b-be0a-491b1b566718
# ╟─ea2eeac2-5514-4fe8-8921-6235cdb0d1ad
# ╟─2bf13604-71c6-474f-90de-5b270c5c0708
# ╠═743b31cd-6b2a-472c-8a70-a3955a2c26b6
# ╟─03bd6ff6-107f-4b04-9cb8-fe5c28570e58
# ╠═08fc4616-d2af-471b-adc0-195bb566334f
# ╠═d0c1e91e-d1df-4a12-a0e2-8513eeee7ba6
# ╠═8623c9b7-5724-4678-9b53-4312389a276d
# ╠═da0220ea-99aa-4a0d-8826-f0e461fd503e
# ╟─60d1ea9b-b935-4e88-9e92-07497f06aeaa
# ╟─efbb46be-c442-4ab3-85a0-b10cbc3a2a9e
# ╠═8b2ec98a-b37f-4bd0-9fc8-9f3dcf9550c9
# ╠═25c22a1f-6ae4-4b72-8e4c-de844e2699d1
# ╟─582b74c7-aaf6-4a8e-901f-489199ccc0d6
# ╠═3e575a45-50d6-4297-84e5-6699e36d00dd
# ╟─ed1de34a-a42e-474d-ba36-411163a343f8
# ╠═947c169c-a900-4728-ad29-5200b52f9599
# ╠═04bbc594-17c9-4b41-a599-d41e2e541c9f
# ╟─29a0f8c8-5529-496c-99f1-b436d15f2517
# ╟─f346a0b3-4d1d-49ad-94ec-cc42ee75a05b
# ╠═8a1c55e5-1008-44ee-b853-c53c2dbf3bb2
# ╠═2ee789c5-ae38-4e85-95e1-920a580df058
# ╠═9d1f3408-e0f5-4093-bb6b-862485e99476
# ╟─a4d4a26f-ff25-4bf0-b8ab-2db56131ee3e
# ╟─91f1ce37-0d68-4ed2-9359-dbcfd06f6cdd
# ╠═4328e814-82a8-4e7d-8b8a-44a9d5ba17c5
# ╟─7b11ac9b-a470-4059-bbad-ba6bc443c68b
# ╠═a5629c28-53bf-4b19-a49f-ab26df1d1b46
# ╠═74adb98a-1ad5-4181-a3be-ec5fe2565a35
# ╟─c38e202f-a2d2-4449-8308-df22774b7af1
# ╟─d5ab9847-ace4-4a5b-8c81-d2eb8b56c896
# ╟─575e3106-704e-4bea-8155-7ee6e6c9da5d
# ╠═bff34b78-f7ad-452e-9b95-a498f3e377f5
# ╟─bba5eaa1-16f2-401d-8cf3-f05e9761583d
# ╠═ddbb84d1-945a-4184-850b-31ee3540e960
# ╟─f0215a78-146d-4009-ae73-8a2f35072db9
# ╠═84d86795-e2c4-4c2c-888b-ae999bcda2dd
# ╠═8c592351-4da7-4bed-a99c-595156c1ed8b
# ╠═e3411cdb-acf7-40da-9f5e-6852c1370797
# ╠═5fbfe4aa-f2e5-4845-92e7-9de00063059f
# ╟─ffecadd1-95b5-412f-86e4-e4dee733cab1
# ╠═bad93338-72e3-46fc-888f-e507c62cefc6
# ╟─63c14c9e-808e-464b-b5cc-25921aa34f56
# ╠═8f8a9dec-4de2-4fc7-9d87-a3638ec47ee3
# ╠═e85b9b6d-736d-446a-acb1-b6ed0ccdd907


module Donquicote

using PrettyTables, Statistics, StatsBase, ForwardDiff

D(f) = x -> ForwardDiff.derivative(f,float(x))

function newton(f, ∂f, x0; xatol = 1e-7)

    diff = 1
    while diff >= xatol
        x1 = x0 - f(x0) / ∂f(x0);
        diff = abs(x1 - x0);
        x0 = x1;
    end
    return x0
end

function haley(f, ∂f, ∂2f, x0; xatol = 1e-7)
    diff = 1
    while diff >= xatol
        x1 = x0 - (2*f(x0)*∂f(x0)) / (2*∂f(x0)^2 - f(x0)*∂2f(x0));
        diff = abs(x1 - x0);
        x0 = x1;
    end
    return x0

end

function schroeder(f, ∂f, ∂2f, x0; xatol = 1e-7)
    diff = 1
    while diff >= xatol
        x1 = x0 - f(x0) / ∂f(x0)  - (∂2f(x0)*f(x0)^2)/(2*∂f(x0)^3);
        diff = abs(x1 - x0);
        x0 = x1;
    end
    return x0

end

function weightedmean(x, w)
    wxsum = wsum = 0
    for (x,w) in zip(x,w)
        wx = w*x
        if !ismissing(wx) | isfinite(wx)
            wxsum += wx
            wsum += w
        end
    end
    return wxsum / wsum
end

function weightedstd(x, w)
    μ = weightedmean(x, w)
    wxsum = wsum = 0
    for (x,w) in zip(x,w)
        wx = w*(x - μ)^2
        if !ismissing(wx) | isfinite(wx)
            wxsum += wx
            wsum += w
        end
    end
    σ = sqrt((1/wsum)*wxsum)
    return σ
end

function gini(x::Vector{Float64})
    n = length(x)
    sarray = sort(x)
    return 2*(sum(collect(1:n).*sarray))/(n*sum(sarray))-1
end

logm(x) = x > 0.0 ? log(x) : missing

lvar(x) = var(logm.(x))

nanmean(x) = mean(filter(!isnan,x))

format(x) = round(x, digits = 3)

function savehtml(filename, data)
    open("$filename.html", "w") do f
        pretty_table(f, data, backend = :html)
    end
end

export newton, haley, schroeder
export gini, nanmean, weightedmean, weightedstd, format, logm, lvar
export ones, zeros
export savehtml

end


using NumericalMethodsforEngineers

#=
@sym begin
  LagrangePolynomial(xi_, yi_) := Module([N, sum, num, den],
    begin
      N = Length(xi)
      sum = 0
      For( i=1, i <= N, Increment(i),
        begin
          num = 1
          den = 1
          For( j=1, j <= N, Increment(j),
            begin
              If(j != i, num *= (x-xi[j]))
              If(j != i, den *= (xi[i]-xi[j]))
            end
          )
          sum += yi[i] * num/den
        end
      ),
      Return(Simplify(sum))
    end
  )
end
=#

println("\n\n--------1------------\n")

@sym begin
  ClearAll(xi, yi, Y, R, sol, ytilde)
  xi = [0, 1//2, 1]
  yi = [0, a, 1]
  Y(x_) := LagrangePolynomial(xi, yi)
  Println(Y(x))
end

println("\n---------2-----------\n")

@sym begin
  ClearAll(xi, yi, Y1, R, sol, ytilde)
  Y1(x_) := x^3 -3x^2 +6x - 9
  R(x_) = Simplify(D(Y1(x), x, 2))
  Println(R(x))
end

println("\n----------3----------\n")

@sym begin
  ClearAll(xi, yi, Y, R, sol, ytilde)
  xi = [0, 1//2, 1]
  yi = [0, a, 1]
  Y(x_) := LagrangePolynomial(xi, yi)
  R(x_) = Simplify(D(Y(x), x, 2) - 3*x - 4*Y(x))
  R(x_) = Simplify(R(x) ./ (a => -C1/4))
  Println(R(x))
  SetJ(r, ToString(Simplify(R(x))))
end

println("\n----------4----------\n")

println(r)

println("\n----------5----------\n")

println("\n\nExample 7.14: y'' = 3x + 4y, y(0)=0, y(1)=1")
println("Residual for Weighted Residual Method using 2 point Lagragian Polynomial")
@sym Println("\nY(x): ", Expand(Y(x)), "\n")
println("( Example 7.14 gives: ytilde = 1/2*x^3*(27a-27b+9)-x^2*(45a-36b+9)+x*(18a-9b+2) )")
@sym Println("\nR(x) = ", R(x), "\n")


### A Pluto.jl notebook ###
# v0.12.10

using Markdown
using InteractiveUtils

# ╔═╡ e077ab16-22e5-11eb-15ed-5f0654bf7328
using Pkg, DrWatson

# ╔═╡ e75a9a06-22e5-11eb-2e36-8d4f62b830ed
begin
	@quickactivate "StatisticsWithJuliaPlutoNotebooks"
	using Random, Distributions, StatsPlots, Plots, Measures
	Random.seed!(0)
 end

# ╔═╡ ca5fbdc8-22e5-11eb-0a60-cb0a127a3ca5
md"## Listing4.21"

# ╔═╡ fa68607e-22e5-11eb-0558-c9a4d9f77426
begin
	b1, b2 = 0.5 , 2
	dist1, dist2, = Beta(b1,b1), Beta(b2,b2)

	n = 2000
	data1 = rand(dist1,n)
	data2 = rand(dist2,n)
end

# ╔═╡ 65067674-250d-11eb-12ab-f7aeb1fb1a16
begin
	stephist(data1, bins=15, label = "beta($b1,$b1)", c = :red, normed = true)
	p1 = stephist!(data2, bins=15, label = "beta($b2,$b2)",
			c = :blue, xlabel="x", ylabel="Density",normed = true)

	p2 = qqplot(data1, data2, c=:black, ms=1, msw =0,
			xlabel="Quantiles for beta($b1,$b1) sample",
			ylabel="Quantiles for beta($b2,$b2) sample",
			legend=false)

	plot(p1, p2, size=(800,400), margin = 5mm)
end

# ╔═╡ 475ff888-22e6-11eb-2354-09f8f40a8e12
md"## End of listing4.21"

# ╔═╡ Cell order:
# ╟─ca5fbdc8-22e5-11eb-0a60-cb0a127a3ca5
# ╠═e077ab16-22e5-11eb-15ed-5f0654bf7328
# ╠═e75a9a06-22e5-11eb-2e36-8d4f62b830ed
# ╠═fa68607e-22e5-11eb-0558-c9a4d9f77426
# ╠═65067674-250d-11eb-12ab-f7aeb1fb1a16
# ╟─475ff888-22e6-11eb-2354-09f8f40a8e12