# Copyright (c) 2019 Arpit Bhatia and contributors                               #src
#                                                                                #src
# Permission is hereby granted, free of charge, to any person obtaining a copy   #src
# of this software and associated documentation files (the "Software"), to deal  #src
# in the Software without restriction, including without limitation the rights   #src
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell      #src
# copies of the Software, and to permit persons to whom the Software is          #src
# furnished to do so, subject to the following conditions:                       #src
#                                                                                #src
# The above copyright notice and this permission notice shall be included in all #src
# copies or substantial portions of the Software.                                #src
#                                                                                #src
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR     #src
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,       #src
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE    #src
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER         #src
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,  #src
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE  #src
# SOFTWARE.                                                                      #src

# # Variables, constraints, and objective functions

# **Originally Contributed by**: Arpit Bhatia

# While the last tutorial introduced you to basics of of JuMP code, this
# tutorial will go in depth focusing on how to work with different parts of a
# JuMP program.

using JuMP
model = Model();

# ## Variable basics

# ### Variable bounds

# All of the variables we have created till now have had a bound. We can also
# create a free variable.

@variable(model, free_x)

# While creating a variable, instead of using the <= and >= syntax, we can also
# use the `lower_bound` and `upper_bound` keyword arguments.

@variable(model, keyword_x, lower_bound = 1, upper_bound = 2)

# We can query whether a variable has a bound using the `has_lower_bound` and
# `has_upper_bound` functions. The values of the bound can be obtained using the
# `lower_bound` and `upper_bound` functions.

has_upper_bound(keyword_x)

#-

upper_bound(keyword_x)

# Note querying the value of a bound that does not exist will result in an error.

try #hide
lower_bound(free_x)
catch err; showerror(stderr, err); end  #hide

# JuMP also allows us to change the bounds on variable. We will learn this in
# the problem modification tutorial.

# ### [Containers](@id tutorial_variable_container)

# We have already seen how to add a single variable to a model using the
# [`@variable`](@ref) macro. Let's now look at more ways to add variables to a
# JuMP model.

# JuMP provides data structures for adding collections of variables to a model.
# These data structures are referred to as Containers and are of three types:
# `Arrays`, `DenseAxisArrays`, and `SparseAxisArrays`.

# #### Arrays

# JuMP arrays are created in a similar syntax to Julia arrays with the addition
# of specifying that the indices start with 1. If we do not tell JuMP that the
# indices start at 1, it will create a `DenseAxisArray` instead.

@variable(model, a[1:2, 1:2])

# An n-dimensional variable $x \in {R}^n$ having a bound $l \preceq x \preceq u$
# ($l, u \in {R}^n$) is added in the following manner.

n = 10
l = [1; 2; 3; 4; 5; 6; 7; 8; 9; 10]
u = [10; 11; 12; 13; 14; 15; 16; 17; 18; 19]

@variable(model, l[i] <= x[i = 1:n] <= u[i])

# Note that while working with Containers, we can also create variable bounds
# depending upon the indices:

@variable(model, y[i = 1:2, j = 1:2] >= 2i + j)

# #### DenseAxisArrays

# `DenseAxisArrays` are used when the required indices are not one-based integer
# ranges. The syntax is similar except with an arbitrary vector as an index as
# opposed to a one-based range.

# An example where the indices are integers but do not start with one.

@variable(model, z[i = 2:3, j = 1:2:3] >= 0)

# Another example where the indices are an arbitrary vector.

@variable(model, w[1:5,["red", "blue"]] <= 1)

# #### SparseAxisArrays

# `SparseAxisArrays` are created when the indices do not form a rectangular set.
# For example, this applies when indices have a dependence upon previous indices
# (called triangular indexing).

@variable(model, u[i = 1:3, j = i:5])

# We can also conditionally create variables by adding a comparison check that
# depends upon the named indices and is separated from the indices by a
# semi-colon (;).

@variable(model, v[i = 1:9; mod(i, 3) == 0])

# ### Variable types

# The last arguement to the `@variable` macro is usually the variable type. Here
# we'll look at how to specifiy the variable type.

# #### Integer variables

# Integer optimization variables are constrained to the set $x \in {Z}$

@variable(model, integer_x, Int)

# or

@variable(model, integer_z, integer = true)

# #### Binary variables

# Binary optimization variables are constrained to the set $x \in \{0, 1\}$.

@variable(model, binary_x, Bin)

# or

@variable(model, binary_z, binary = true)

# #### Semidefinite variables

# JuMP also supports modeling with semidefinite variables. A square symmetric
# matrix X is positive semidefinite if all eigenvalues are nonnegative.

@variable(model, psd_x[1:2, 1:2], PSD)

# We can also impose a weaker constraint that the square matrix is only
# symmetric (instead of positive semidefinite) as follows:

@variable(model, sym_x[1:2, 1:2], Symmetric)

# ## Constraint basics

model = Model()
@variable(model, x)
@variable(model, y)
@variable(model, z[1:10]);

# ### Constraint references

# While calling the `@constraint` macro, we can also set up a constraint
# reference. Such a referrence is useful for obtaining additional information
# about the constraint, such as its dual solution.

@constraint(model, con, x <= 4)

# ### [Containers](@id tutorial_constraint_container)

# Just as we had containers for variables, JuMP also provides `Arrays`,
# `DenseAxisArrays`, and `SparseAxisArrays` for storing collections of
# constraints. Examples for each container type are given below.

# #### Arrays

@constraint(model, [i = 1:3], i * x <= i + 1)

# #### DenseAxisArrays

@constraint(model, [i = 1:2, j = 2:3], i * x <= j + 1)

# #### SparseAxisArrays

@constraint(model, [i = 1:2, j = 1:2; i != j], i * x <= j + 1)

# ### Constraints in a loop

# We can add constraints using regular Julia loops

for i in 1:3
    @constraint(model, 6x + 4y >= 5i)
end

# or use for each loops inside the `@constraint` macro.

@constraint(model, [i in 1:3], 6x + 4y >= 5i)

# We can also create constraints such as $\sum _{i = 1}^{10} z_i \leq 1$

@constraint(model, sum(z[i] for i in 1:10) <= 1)

# ## Objective functions

# While the recommended way to set the objective is with the [`@objective`](@ref)
# macro, the functions [`set_objective_sense`](@ref) and [`set_objective_function`](@ref)
# provide an equivalent lower-level interface.

using GLPK

model = Model(GLPK.Optimizer)
@variable(model, x >= 0)
@variable(model, y >= 0)
set_objective_sense(model, MOI.MIN_SENSE)
set_objective_function(model, x + y)

optimize!(model)

#-

objective_value(model)

# To query the objective function from a model, we use the [`objective_sense`](@ref),
# [`objective_function`](@ref), and [`objective_function_type`](@ref) functions.

objective_sense(model)

#-

objective_function(model)

#-

objective_function_type(model)

# # Vectorized syntax

# We can also add constraints and an objective to JuMP using vectorized linear
# algebra. We'll illustrate this by solving an LP in standard form i.e.

# ```math
# \begin{aligned}
# & \min & c^T x \\
# & \;\;\text{s.t.} & A x = b \\
# & & x \succeq 0 \\
# & & x \in \mathbb{R}^n
# \end{aligned}
# ```

vector_model = Model(GLPK.Optimizer)

A= [ 1 1 9  5;
     3 5 0  8;
     2 0 6 13]

b = [7; 3; 5]

c = [1; 3; 5; 2]

@variable(vector_model, x[1:4] >= 0)
@constraint(vector_model, A * x .== b)
@objective(vector_model, Min, c' * x)

optimize!(vector_model)

#-

objective_value(vector_model)


#' # MNIST: Knet.jl CNN
info("MNIST: Knet.jl CNN example") #jl-only

#' In this tutorial we will adapt the
#' [MNIST example](http://denizyuret.github.io/Knet.jl/latest/tutorial.html#Convolutional-neural-network-1)
#' from [Knet.jl](https://github.com/denizyuret/Knet.jl)
#' to utilize a custom augmentation pipeline.
#' In order to showcase the effect that image augmentation can
#' have on a neural network's ability to generalize, we will
#' limit the training set to just the first 500 images (of the
#' available 60,000!). For more information on the dataset see
#' [^MNIST1998].

#md #' !!! note
#md #'
#md #'     This tutorial is also available as a
#md #'     [Juypter](https://jupyter.org/) notebook. You can
#md #'     find a link to the Juypter version of this tutorial
#md #'     in the top right corner of this page.

#' ## Preparing the MNIST dataset
info("Preparing the MNIST dataset") #jl-only

#' In order to access, prepare, and visualize the MNIST images we
#' employ the help of three additional Julia packages. In the
#' interest of time and space we will not go into great detail
#' about their functionality. Feel free to click on their
#' respective names to find out more information about the
#' utility they can provide.
#'
#' - [MLDatasets.jl](https://github.com/JuliaML/MLDatasets.jl)
#'   has an MNIST submodule that offers a convenience interface
#'   to read the MNIST database.
#'
#' - [Images.jl](https://github.com/JuliaImages/Images.jl) will
#'   provide us with the necessary tools to process and display
#'   the image data in Julia / Juypter.
#'
#' - [MLDataUtils.jl](https://github.com/JuliaML/MLDataUtils.jl)
#'   implements a variety of functions to convert and partition
#'   Machine Learning datasets. This will help us prepare the
#'   MNIST data to be used with Knet.jl.

using Images, MLDatasets, MLDataUtils, Random
Random.seed!(42);
#md nothing # hide

#' As you may have seen previously in the
#' [elastic distortions tutorial](@ref elastic), the function
#' `MNIST.traintensor` returns the MNIST training images
#' corresponding to the given indices as a multi-dimensional
#' array. These images are stored in the native horizontal-major
#' memory layout as a single array. Because we specify that
#' the `eltype` of that array should be `Float32`, all the
#' individual values are scaled to be between `0.0` and `1.0`.
#' Also note, how the observations are laid out along the last
#' array dimension

@show summary(MNIST.traintensor(Float32, 1:500));
#md nothing # hide

#' The corresponding label of each image is stored as an integer
#' value between `0` and `9`. That means that if the label has
#' the value `3`, then the corresponding image is known to be a
#' handwritten "3". To show a more concrete example, the
#' following code reveals that the first training image denotes a
#' "5" and the second training image a "0" (etc).

@show summary(MNIST.trainlabels(1:500))
println("First eight labels: ", join(MNIST.trainlabels(1:8),", "))

#' For Knet we will require a slightly format for the images
#' and also the labels. More specifically, we add an additional
#' singleton dimension of length 1 to our image array. Think of
#' this as our single color channel (because MNIST images are gray).
#' Additionally we will convert our labels to proper 1-based indices.
#' This is because some functions provided by Knet expect the labels
#' to be in this format. We will do all this by creating a little
#' utility function that we will name `prepare_mnist`.

"""
    prepare_mnist(images, labels) -> (X, Y)

Change the dimension layout x1×x2×N of the given array
`images` to x1×x2×1×N and return the result as `X`.
The given integer vector `labels` is transformed into
an integer vector denoting 1-based class indices.
"""
function prepare_mnist(images, labels)
    X = reshape(images, (28, 28, 1, :))
    Y = convertlabel(LabelEnc.Indices{Int8}, labels, 0:9)
    X, Y
end
#md nothing # hide

#' With `prepare_mnist` defined, we can now use it in conjunction
#' with the functions in the `MLDatasets.MNIST` sub-module to load
#' and prepare our training set. Recall that for this tutorial only
#' the first 500 images of the training set will be used.

train_x, train_y = prepare_mnist(MNIST.traintensor(Float32, 1:500), MNIST.trainlabels(1:500))
@show summary(train_x) summary(train_y);
[MNIST.convert2image(train_x[:,:,1,i]) for i in 1:8]
#md tmp = hcat(ans...) # hide
#md save("mnist_knet_train.png",repeat(tmp, inner=(4,4))) # hide
#md nothing # hide

#md #' ![training images](mnist_knet_train.png)

#' Similarly, we use `MNIST.testtensor` and `MNIST.testlabels`
#' to load the full MNIST test set. We will utilize that data to
#' measure how well the network is able to generalize with and
#' without augmentation.

test_x, test_y = prepare_mnist(MNIST.testtensor(Float32), MNIST.testlabels())
@show summary(test_x) summary(test_y);
[MNIST.convert2image(test_x[:,:,1,i]) for i in 1:8]
#md tmp = hcat(ans...) # hide
#md save("mnist_knet_test.png",repeat(tmp, inner=(4,4))) # hide
#md nothing # hide

#md #' ![test images](mnist_knet_test.png)

#' ## Defining the Network
info("Defining the Network") #jl-only

#' With the dataset prepared, we can now define and instantiate
#' our neural network. To keep things simple, we will use the same
#' convolutional network as defined in the
#' [MNIST example](http://denizyuret.github.io/Knet.jl/latest/tutorial.html#Convolutional-neural-network-1)
#' of the Knet.jl package.

using Knet
#md nothing # hide

#' The first thing we will do is define the forward pass through
#' the network. This will effectively outline the computation
#' graph of the network architecture. Note how this does not
#' define some details, such as the number of neurons per layer.
#' We will define those later when initializing our
#' vector of weight arrays `w`.

"""
    forward(w, x) -> a

Compute the forward pass for the given minibatch `x` by using the
neural network parameters in `w`. The resulting (unnormalized)
activations of the last layer are returned as `a`.
"""
function forward(w, x)
    # conv1 (2x2 maxpool)
    a1 = pool(relu.(conv4(w[1], x)  .+ w[2]))
    # conv2 (2x2 maxpool)
    a2 = pool(relu.(conv4(w[3], a1) .+ w[4]))
    # dense1 (relu)
    a3 = relu.(w[5] * mat(a2) .+ w[6])
    # dense2 (identity)
    a4 = w[7] * a3 .+ w[8]
    return a4
end
#md nothing # hide

#' In order to be able to train our network we need to choose a
#' cost function. Because this is a classification problem we will
#' use the negative log-likelihood (provided by `Knet.nll`).
#' With the cost function defined we can the simply use the
#' higher-order function `grad` to create a new function `costgrad`
#' that computes us the corresponding gradients.

"""
    cost(w, x, y) -> AbstractFloat

Compute the per-instance negative log-likelihood for the data
in the minibatch `(x, y)` given the network with the current
parameters in `w`.
"""
cost(w, x, y) = nll(forward(w, x), y)
costgrad = grad(cost)
#md nothing # hide

#' Aside from the cost function that we need for training, we
#' would also like a more interpretable performance measurement.
#' In this tutorial we will use "accuracy" for its simplicity
#' and because we know that the class distribution for MNIST
#' is close to uniform.

"""
    acc(w, X, Y; [batchsize]) -> Float64

Compute the accuracy for the data in `(X,Y)` given the network
with the current parameters in `w`. The resulting value is
computed by iterating over the data in minibatches of size
`batchsize`.
"""
function acc(w, X, Y; batchsize = 100)
    sum = 0; count = 0
    for (x_cpu, y) in eachbatch((X, Y), maxsize = batchsize)
        x = KnetArray{Float32}(x_cpu)
        sum += Int(accuracy(forward(w,x), y, average = false))
        count += length(y)
    end
    return sum / count
end
#md nothing # hide

#' Before we can train or even just use our network, we need to
#' define how we initialize `w`, which is our the vector of
#' parameter arrays. The dimensions of these individual arrays
#' specify the filter sizes and number of neurons.
#' It can be helpful to compare the indices here with the indices
#' used in our `forward` function to see which array corresponds
#' to which computation node of our network.

function weights(atype = KnetArray{Float32})
    w = Array{Any}(undef, 8)
    # conv1
    w[1] = xavier(5,5,1,20)
    w[2] = zeros(1,1,20,1)
    # conv2
    w[3] = xavier(5,5,20,50)
    w[4] = zeros(1,1,50,1)
    # dense1
    w[5] = xavier(500,800)
    w[6] = zeros(500,1)
    # dense2
    w[7] = xavier(10,500)
    w[8] = zeros(10,1)
    return map(a->convert(atype,a), w)
end
#md nothing # hide

#' ## Training without Augmentation
info("Training baseline network without augmentation") #jl-only

#' In order to get an intuition for how useful augmentation can
#' be, we need a sensible baseline to compare to. To that end, we
#' will first train the network we just defined using only the
#' (unaltered) 500 training examples.

#' The package
#' [ValueHistories.jl](https://github.com/JuliaML/ValueHistories.jl)
#' will help us record the accuracy during the training process.
#' We will use those logs later to visualize the differences
#' between having augmentation or no augmentation.

using ValueHistories
using ProgressMeter #jl-only

#' To keep things simple, we will not overly optimize our
#' training function. Thus, we will be content with using a
#' closure. Because both, the baseline and the augmented version,
#' will share this "inefficiency", we should still get a decent
#' enough picture of their performance differences.

function train_baseline(; epochs = 500, batchsize = 100, lr = .03)
    w = weights()
    log = MVHistory()
    p = Progress(epochs, desc = "Baseline: ") #jl-only
    for epoch in 1:epochs
        for (batch_x_cpu, batch_y) in eachbatch((train_x ,train_y), batchsize)
            batch_x = KnetArray{Float32}(batch_x_cpu)
            g = costgrad(w, batch_x, batch_y)
            Knet.update!(w, g, lr = lr)
        end

        next!(p) #jl-only
        if (epoch % 5) == 0
            train = acc(w, train_x, train_y)
            test  = acc(w, test_x,  test_y)
            @trace log epoch train test
            msg = "epoch " * lpad(epoch,4) * ": train accuracy " * rpad(round(train, digits=3),5,"0") * ", test accuracy " * rpad(round(test,digits=3),5,"0")
            cancel(p, msg, :blue) #jl-only
#md             println(msg)
#jp             println(msg)
        end
    end
    finish!(p) #jl-only
    log
end
#md nothing # hide

#' Aside from the accuracy, we will also keep an eye on the
#' training time. In particular we would like to see if and how
#' the addition of augmentation causes our training time to
#' increase.

train_baseline(epochs=1) # warm-up
baseline_log = @time train_baseline(epochs=200);
#md nothing # hide

#' As we can see, the accuracy on the training set is around a
#' 100%, while the accuracy on the test set peaks around 90%. For
#' a mere 500 training examples, this isn't actually that bad of
#' a result.

#' ## Integrating Augmentor
info("Training network with augmentation") #jl-only

#' Now that we have a network architecture with a baseline to
#' compare to, let us finally see what it takes to add Augmentor
#' to our experiment. First, we need to include the package to
#' our experiment.

using Augmentor

#' The next step, and maybe the most human-hour consuming part of
#' adding image augmentation to a prediction problem, is to
#' design and select a sensible augmentation pipeline. Take a
#' look at the [elastic distortions tutorial](@ref elastic) for
#' an example of how to do just that.

#' For this example, we already choose a quite complicated but
#' promising augmentation pipeline for you. This pipeline was
#' designed to yield a large variation of effects as well as to
#' showcase how even deep pipelines are quite efficient in terms
#' of performance.

pl = Reshape(28,28) |>
     PermuteDims(2,1) |>
     ShearX(-5:5) * ShearY(-5:5) |>
     Rotate(-15:15) |>
     CropSize(28,28) |>
     Zoom(0.9:0.1:1.2) |>
     CacheImage() |>
     ElasticDistortion(10) |>
     PermuteDims(2,1) |>
     Reshape(28,28,1)
println(pl) #jl-only

#' Most of the used operations are quite self explanatory, but
#' there are some details about this pipeline worth pointing out
#' explicitly.

#' 1. We use the operation [`PermuteDims`](@ref) to convert the
#'    horizontal-major MNIST image to a julia-native
#'    vertical-major image. The vertical-major image is then
#'    processed and converted back to a horizontal-major array.
#'    We mainly do this here to showcase the option, but it is
#'    also to keep consistent with how the data is usually used
#'    in the literature. Alternatively, one could just work with
#'    the MNIST data in a vertical-major format all the way
#'    through without any issue.

#' 2. As counter-intuitive as it sounds, the operation
#'    [`CacheImage`](@ref) right before
#'    [`ElasticDistortion`](@ref) is actually used to improve
#'    performance. If we were to omit it, then the whole pipeline
#'    would be applied in one single pass. In this case, applying
#'    distortions on top of affine transformations lazily is in
#'    fact less efficient than using a temporary variable.

#' With the pipeline now defined, let us quickly peek at what
#' kind of effects we can achieve with it. In particular, lets
#' apply the pipeline multiple times to the first training image
#' and look at what kind of results it produces.

#jp [MNIST.convert2image(reshape(augment(train_x[:,:,:,1], pl), (28, 28))) for i in 1:2, j in 1:8]
#md [MNIST.convert2image(reshape(augment(train_x[:,:,:,1], pl), (28, 28))) for i in 1:8, j in 1:2]
#md tmp = vcat(hcat(ans[:,1]...), hcat(ans[:,2]...)) # hide
#md save("mnist_knet_aug.png",repeat(tmp, inner=(4,4))) # hide
#md nothing # hide

#md #' ![augmented samples](mnist_knet_aug.png)

#' As we can see, we can achieve a wide range of effects, from
#' more subtle to more pronounced. The important part is that all
#' examples are still clearly representative of the true label.

#' Next, we have to adapt the function `train_baseline` to make
#' use of our augmentation pipeline. To integrate Augmentor
#' efficiently, there are three necessary changes we have to
#' make.

#' 1. Preallocate a buffer with the same size and element type
#'    that each batch has.
#'
#'    ```
#'    batch_x_aug = zeros(Float32, 28, 28, 1, batchsize)
#'    ```

#' 2. Add a call to [`augmentbatch!`](@ref) in the inner loop of
#'    the batch iterator using our pipeline and buffer.
#'
#'    ```
#'    augmentbatch!(batch_x_aug, batch_x_org, pl)
#'    ```

#' 3. Replace `batch_x_org` with `batch_x_aug` in the constructor
#'    of `KnetArray`.
#'
#'    ```
#'    batch_x = KnetArray{Float32}(batch_x_aug)
#'    ```

#' Applying these changes to our `train_baseline` function
#' will give us something similar to the following function.
#' Note how all the other parts of the function remain exactly
#' the same as before.

function train_augmented(; epochs = 500, batchsize = 100, lr = .03)
    w = weights()
    log = MVHistory()
    p = Progress(epochs, desc = "Augmented: ") #jl-only
    batch_x_aug = zeros(Float32, size(train_x,1), size(train_x,2), 1, batchsize)
    for epoch in 1:epochs
        for (batch_x_cpu, batch_y) in eachbatch((train_x ,train_y), batchsize)
            augmentbatch!(CPUThreads(), batch_x_aug, batch_x_cpu, pl)
            batch_x = KnetArray{Float32}(batch_x_aug)
            g = costgrad(w, batch_x, batch_y)
            Knet.update!(w, g, lr = lr)
        end

        next!(p) #jl-only
        if (epoch % 5) == 0
            train = acc(w, train_x, train_y)
            test  = acc(w, test_x,  test_y)
            @trace log epoch train test
            msg = "epoch " * lpad(epoch,4) * ": train accuracy " * rpad(round(train, digits=3),5,"0") * ", test accuracy " * rpad(round(test,digits=3),5,"0")
            cancel(p, msg, :blue) #jl-only
#md             println(msg)
#jp             println(msg)
        end
    end
    finish!(p) #jl-only
    log
end
#md nothing # hide

#' You may have noticed in the code above that we also pass a
#' `CPUThreads()` as the first argument to [`augmentbatch!`](@ref).
#' This instructs Augmentor to process the images of the batch in
#' parallel using multi-threading. For this to work properly you
#' will need to set the environment variable `JULIA_NUM_THREADS`
#' to the number of threads you wish to use. You can check how
#' many threads are used with the function `Threads.nthreads()`

@show Threads.nthreads();
#md nothing # hide

#' Now that all pieces are in place, let us train our network
#' once more. We will use the same parameters except that now
#' instead of the original training images we will be using
#' randomly augmented images. This will cause every epoch to be
#' different.

train_augmented(epochs=1) # warm-up
augmented_log = @time train_augmented(epochs=200);
#md nothing # hide

#' As we can see, our network reaches far better results on our
#' testset than our baseline network did. However, we can also
#' see that the training took quite a bit longer than before.
#' This difference generally decreases as the complexity of the
#' utilized neural network increases. Yet another way to improve
#' performance (aside from simplifying the augmentation pipeline)
#' would be to increase the number of available threads.

#' ## Improving Performance
info("Improving Performance") #jl-only

#' One of the most effective ways to make the most out of the
#' available resources is to augment the next (couple) mini-batch
#' while the current minibatch is being processed on the GPU.
#' We can do this via julia's build in parallel computing
#' capabilities

#' First we need a worker process that will be responsible for
#' augmenting our dataset each epoch. This worker also needs
#' access to a couple of our packages

using Distributed
addprocs(1)
@everywhere using Augmentor, MLDataUtils
#md nothing # hide

#' Next, we replace the inner `eachbatch` loop with a more
#' complicated version using a `RemoteChannel` to exchange and
#' queue the augmented data.

function async_train_augmented(; epochs = 500, batchsize = 100, lr = .03)
    w = weights()
    log = MVHistory()
    p = Progress(epochs, desc = "Async Augmented: ") #jl-only
    for epoch in 1:epochs
        @sync begin
            local_ch = Channel{Tuple}(4) # prepare up to 4 minibatches in adavnce
            remote_ch = RemoteChannel(()->local_ch)
            @spawn begin
                # This block is executed on the worker process
                batch_x_aug = zeros(Float32, size(train_x,1), size(train_x,2), 1, batchsize)
                for (batch_x_cpu, batch_y) in eachbatch((train_x ,train_y), batchsize)
                    # we are still using multithreading
                    augmentbatch!(CPUThreads(), batch_x_aug, batch_x_cpu, pl)
                    put!(remote_ch, (batch_x_aug, batch_y))
                end
                close(remote_ch)
            end
            @async begin
                # This block is executed on the main process
                for (batch_x_aug, batch_y) in local_ch
                    batch_x = KnetArray{Float32}(batch_x_aug)
                    g = costgrad(w, batch_x, batch_y)
                    Knet.update!(w, g, lr = lr)
                end
            end
        end

        next!(p) #jl-only
        if (epoch % 5) == 0
            train = acc(w, train_x, train_y)
            test  = acc(w, test_x,  test_y)
            @trace log epoch train test
            msg = "epoch " * lpad(epoch,4) * ": train accuracy " * rpad(round(train,digits=3),5,"0") * ", test accuracy " * rpad(round(test,digits=3),5,"0")
            cancel(p, msg, :blue) #jl-only
#md             println(msg)
#jp             println(msg)
        end
    end
    finish!(p) #jl-only
    log
end
#md nothing # hide

#' Note that for this toy example the overhead of this approach
#' is greater than the benefit.

async_train_augmented(epochs=1) # warm-up
augmented_log = @time async_train_augmented(epochs=200);
#md nothing # hide

#' ## Visualizing the Results
info("Visualizing the Results") #jl-only

#' Before we end this tutorial, let us make use the
#' [Plots.jl](https://github.com/JuliaPlots/Plots.jl) package to
#' visualize and discuss the recorded training curves.
#' We will plot the accuracy curves of both networks side by side
#' in order to get a good feeling about their differences.

using Plots
#jp gr()
#md gr()
#md nothing # hide
unicodeplots() #jl-only

#+

#md default(bg_outside=colorant"#FFFFFF") # hide
plt = plot(
    plot(baseline_log,  title="Baseline",  ylim=(.5,1)),
    plot(augmented_log, title="Augmented", ylim=(.5,1)),
    size = (900, 400),
    xlab = "Epoch",
    ylab = "Accuracy",
    markersize = 1
)
#jp plt
#md png(plt, "mnist_knet_curves.png") # hide
#md nothing # hide

#md #' ![learning curves](mnist_knet_curves.png)

#' Note how the accuracy on the (unaltered) training set
#' increases faster for the baseline network than for the
#' augmented one. This is to be expected, since our augmented
#' network doesn't actually use the unaltered images for
#' training, and thus has not actually seen them. Given this
#' information, it is worth pointing out explicitly how the
#' accuracy on training set is still greater than on the test set
#' for the augmented network as well. This is also not a
#' surprise, given that the augmented images are likely more
#' similar to their original ones than to the test images.

#' For the baseline network, the accuracy on the test set
#' plateaus quite quickly (around 90%). For the augmented network
#' on the other hand, it the accuracy keeps increasing for quite
#' a while longer.

#' ## References
#'
#' [^MNIST1998]: LeCun, Yan, Corinna Cortes, Christopher J.C. Burges. ["The MNIST database of handwritten digits"](http://yann.lecun.com/exdb/mnist/) Website. 1998.


### A Pluto.jl notebook ###
# v0.12.18

using Markdown
using InteractiveUtils

# ╔═╡ a4f0a914-1ec9-11eb-3c22-db87a9776aaa
begin
	using JSON
	using DataFrames
end

# ╔═╡ 5f83510e-1fc5-11eb-3c40-83a844bf2907
using Turing

# ╔═╡ 2414104e-1d41-11eb-37cb-9b5cee78a98b
md"## Creating our conjectures"

# ╔═╡ 3b7f8a1e-1d42-11eb-1ab1-c11fda416e80
md"""If there is one thing we can all agree on, it is that the reality in which we live is complex. The explanation for the things we usually see, and have naturalized in our daily lives, is usually quite complex and requires abstraction from what "is simply seen".

In order to give an explanation and gain a deeper understanding of the things we see, we tend to generate models that seek to explain them in a simple and generalized way. In this way we can reduce the noise of our observations to general rules that "govern" them.

For example, it is obvious to everyone that if we push a glass it will move in the same direction as we did. We also know that if we keep pushing it and it goes beyond the limits of the table, it will fall to the floor. But one thing is to have the intuition of what's going to happen, and another is to have an understanding of the laws that govern that movement. In this case, they are the Newton´s Law´s of motion:

"""

# ╔═╡ 4e4385b4-203b-11eb-0c5b-cbe41156401b
md"$ \vec{F}^{\} = m*\vec{a}^{\} $"

# ╔═╡ 2dca2f42-203b-11eb-2cb3-0d6a3823cdcd
md"""In this way, and with only one formula, it is possible to gain an understanding that is generalizable to many aspects of reality.

#### Observable variables vs Latent variables

Now, it is worth noting that in this case all the variables that make up our model are observable. This means that they can be measured directly.

In the case of a glass, we could weigh it with a scale. Then, by pushing it, we could measure the acceleration it acquired and from these two measurements we could obtain the force we applied to it. So, every parameter of the model is fully defined.

However, as we try to advance in our understanding of reality, we arrive at more and more complex models and many times we are not so lucky to be able to define them with simple observable variables.

For example, this is very common in the economic sciences, where models are created with variables such as "quality of life". Economists will try to measure this latent variable with other variables that can be observed (such as quality of life, schooling rate, number of hospitals for a certain number of inhabitants, etc), but that do not have an obvious and direct relationship as if they had newton's equations.

This type of latent variables are used in different models to gain greater abstraction and to be able to obtain information that a priori is not found at first sight in the data. For example, in the case of economics, from concrete measures of a country's economy it is possible to generalize knowledge and be able to infer an abstract variable such as quality of life.

### Bayesian hierarchical models

The Bayesian framework allows us to build statistical models that can generalize the information obtained from the data and make inferences from latent variables.

A nice way to think about this kind of models is that they allow us to build our "story" about which are the variables that generate the data we are observing. Basically, they allow us to increase the "depth" of our model by indicating that the parameters of our prior distributions also follow other probability distributions.

This sure is sounding very strange. Don't worry, let's move on to an example to clarify it.

#### Football analysis

Let's imagine for a moment that we are brilliant statisticians. We find ourselves looking for new interesting challenges to solve and we come across a sports bookmaker. They tell us that they want to expand into football betting and that they would like us to be able to build a model that allows them to analyze the strengths and weaknesses of English Premier League teams. They are interested because they want to be able to predict possible outcomes and thus be able to price the bets.

The problem is that, as they have never worked in this sector before, they only have the results of the league matches. So what can we do? """

# ╔═╡ 0c0268dc-1ed3-11eb-2f73-b3706305b298
md"We have the data stored in a specific format called JSON, so the first thing to do is to parse and visualize it"

# ╔═╡ cbaaf72c-1eb4-11eb-3509-932b337f270b
begin
england_league = JSON.parsefile("matches_England.json")
matches_df = DataFrame(home = [], away = [], score_home = [], score_away = [])
end;

# ╔═╡ ab6fcb22-1ed2-11eb-2749-3bef16911972
begin
	matches = []
	for match in england_league
		push!(matches, split(match["label"], ","))
end
end

# ╔═╡ d2a81a14-1ed2-11eb-1307-bfb169aebbb4
begin
for match in matches
	home, away = split(match[1], " - ")
	score_home, score_away = split(match[2], " - ")

	push!(matches_df,[home, away, parse(Int,score_home), parse(Int,score_away)])
end
end

# ╔═╡ dd80fc38-1ed2-11eb-0e8c-49859d27c72c
matches_df

# ╔═╡ 7f0c72d4-1f9e-11eb-06f1-cb587d7e5436
teams = unique(collect(matches_df[1]))

# ╔═╡ 868e5c8c-1ed3-11eb-291c-fb26d512c103
md"""So, we have the data of the 380 matches that were played in the Premier League 2017/2018 and our challenge is to be able to analyze the characteristics of these teams.

A priori it may seem that we are missing data, that with the data we have we cannot infer "characteristics" specific to each team. At most, it might be possible to see who the teams that scored the most goals, the averages of goals per game or how the positions were after the tournament, but to obtain characteristics of the teams? how could we face this problem?

#### Creating our stories

Okay, let's see what information we have from our data:
On one hand we have specified the names of each team and which one is local. On the other hand, we have the number of goals scored.

A possible approach to this data is to realize that the goals scored by each team can be modeled with a poisson distribution.

Why? You have to remember that this distribution describes "arrivals" - discrete events - in a continuum. For example, it is widely used to describe customer arrivals to a location as time passes or failures in continuous industrial processes (e.g. failure in the production of a pipe).

In this particular case, we could propose that the goals scored by a team are the discrete events that occur in the time continuum that the game last:

"""

# ╔═╡ 04a8f16c-1ed5-11eb-3979-af35c6d9dea1
md"$Score \sim Poisson(θ)$"

# ╔═╡ 9f1e1618-1edb-11eb-02af-9fa5c8904163
md"""Well, we have an improvement. We've already told our model how to think about goals scored.

Now we can use the flexibility of Bayesianism to indicate what the "goal rate" of our Poisson depends on.  You can think of it literally as the number of goals a team scores per unit of time. And this is where we have to take advantage of all the information provided by the data set.

As expected, this rate has to be particular to each match the team plays and take into account the opponent. We can therefore propose that the scoring rate of each team (in each particular match) depends on the "attacking power" of the team on the one hand, and the "defensive power" of the opponent on the other:

"""

# ╔═╡ 42104900-1f80-11eb-019f-a7c561ca3e4b
md"$θ_{team1} \sim att_{team1} + def_{team2}$"

# ╔═╡ 9d0f3a1e-1f80-11eb-3422-fb7b4696cc13
md"""In this way we could be capturing, from the results of each game, the attack and defence strengths of each team.

Another latent variable that we could obtain, given the data, is if there is an effect that increases (or decreases) the goal rate related to whether the team is local or not. This would also help - in case there is indeed an effect - in not the attack and defence parameters be disrupted by having to "contain" that information."""

# ╔═╡ 0b1f9c68-1f83-11eb-11e7-e7614c021c05
md"$θ_{home} \sim home + att_{home} + def_{away}$
$θ_{away} \sim att_{away} + def_{home}$"

# ╔═╡ 043c438c-1f84-11eb-2cd7-6dd07a125910
md"""This leaves one attack and one defense parameter for each team, and a global league parameter that indicates the effect of being local on the scoring rate.

#### Letting the information flow

Okay, we are already getting much closer to the initial goal we set. As a last step, we must be able to make the information flow between the two independent poissons that we proposed to model the score of each of the two teams that are playing. We need to do that precisely because we have proposed that the poissons are independent, but we need that when making the inference of the parameters the model can access the information from both scores so it can catch the correlation between them. In other words, we have to find a way to interconnect our model.

And that is exactly what hierarchical Bayesian models allow us to do. How? By letting us choose probability distributions for the parameters that represent the characteristics of both equipment. With the addition that these parameters will share the same prior distributions. Let's see how:

The first thing to do, as we already know, is to assign the prior distributions of our attack and defense parameters. A reasonable idea would be to propose that they follow a normal distribution since it is consistent that there are some teams that have a very good defense, so the parameter would take negative values; or there may be others that have a very bad one, taking positive values (since they would "add up" to the goal rate of the opposing team). The normal distribution allows us to contemplate both cases.

Now, when choosing the parameters we are not going to stop and assign fixed numbers, but we will continue to deepen the model and add another layer of distributions:

"""

# ╔═╡ 4a31541e-1f97-11eb-038e-736446224c21
md"$att_{t} \sim Normal(μ_{att}, σ_{att})$
$def_{t} \sim Normal(μ_{def}, σ_{def})$"

# ╔═╡ e3a99868-1f97-11eb-1872-79c8f6abe1be
md"Where the t sub-index is indicating us that there are a couple of these parameters for each team.

Then, as a last step to have our model defined, we have to assign the priority distributions that follow the parameters of each normal distribution. We have to define our hyper priors.

"

# ╔═╡ c3ca1cc4-1f98-11eb-3c23-33acb95fbc03
md"$μ_{att}, μ_{def} \sim Normal(0, 0.1)$
$σ_{att}, σ_{def} \sim Exponential(1)$"

# ╔═╡ 4106e55a-1f99-11eb-110e-b79221774f12
md"We must not forget the parameter that represents the advantage of being local"

# ╔═╡ bff4fdae-1f99-11eb-2d94-aba7bf9d097f
md"$home \sim Normal(0,1)$"

# ╔═╡ 2b135202-1fa7-11eb-324d-93a5eda41e3e
md"""Now that our model is fully define, let's add one last restriction to the characteristics of the teams to make it easier to compare them: subtract the average of all the attack and defence powers from each one. In this way we will have the features centred on zero, with negative values for the teams that have less attacking power than the average and positive values for those that have more. As we already said, the opposite analysis applies to the defence, negative values are the ones that will indicate that a team has a strong defence as they will be "subtracting" from the scoring rate of the opponent. This is equivalent to introducing the restriction:
"""

# ╔═╡ ddc7b166-203a-11eb-0ac2-056c309bb590
md"$\sum att_{t} = 0$
$\sum def_{t} = 0$"

# ╔═╡ 8e164be0-203b-11eb-293a-6b32dba96133
md"Let's translate all this into Turing code:"

# ╔═╡ 51a309bc-2033-11eb-10c0-ed17545df33d
begin
	@model function football_matches(home_teams, away_teams, score_home, score_away, teams)
	#hyper priors
	σatt ~ Exponential(1)
	σdeff ~ Exponential(1)
	μatt ~ Normal(0,0.1)
	μdef ~ Normal(0,0.1)

	home ~ Normal(0,1)

	#Team-specific effects
	att ~ filldist(Normal(μatt, σatt), length(teams))
	def ~ filldist(Normal(μatt, σdeff), length(teams))

	dict = Dict{String, Int64}()
	for (i, team) in enumerate(teams)
		dict[team] = i
	end

	#Zero-sum constrains
	offset = mean(att) + mean(def)

	log_θ_home = Vector{Real}(undef, length(home_teams))
	log_θ_away = Vector{Real}(undef, length(home_teams))

	#Modeling score-rate and scores (as many as there were games in the league)
	for i in 1:length(home_teams)
		#score-rate
		log_θ_home[i] = home + att[dict[home_teams[i]]] + def[dict[away_teams[i]]] - offset
		log_θ_away[i] = att[dict[away_teams[i]]] + def[dict[home_teams[i]]] - offset
		#scores
		score_home[i] ~ LogPoisson(log_θ_home[i])
		score_away[i] ~ LogPoisson(log_θ_away[i])
	end

	end
end

# ╔═╡ f3f0b6f8-2033-11eb-0f9e-d951d791001d
md"""As you can see, the turing code is very clear and direct. In the first block we define our hyperpriors for the distributions of the characteristics of the equipment.

In the second one, we define the priors distributions that will encapsulate the information about the attack and defense powers of the teams. With the *filldist* function we are telling Turing that we need as many of these parameters as there are teams in the league *length(teams)*

Then, we calculate the average of the defense and attack parameters that we are going to use to centralize those variables, and we use the LogPoisson distribution to allow the theta to take some negative value in the inference process and give more sensitivity to the parameters that make it up.

As we said before, we will model the thetas for each game played in the league, that's why the *for* of the last block goes from 1 to *length(home_teams)*, which is the list that contains who was the local team of each game played.

So let´s run it and see if all of this effort was worth it:
"""

# ╔═╡ f3f4bfba-203f-11eb-142c-a187112744d2
model = football_matches(matches_df[1], matches_df[2], matches_df[3], matches_df[4], teams)

# ╔═╡ 2ce4435c-2040-11eb-1670-e39ad4cc690c
posterior = sample(model, NUTS(),3000);

# ╔═╡ 9e50e56e-5674-11eb-2919-4bde8964bc59
rand(LogPoisson(0.2), 100)

# ╔═╡ 8c1dd9de-2040-11eb-35a3-39e8196577a1
md"#### Analyzing the results
In order to compare and corroborate that the inference of our model makes sense, it is key to have the ranking table of how the teams actually performed in the 2017/2018 Premier League.
"

# ╔═╡ 0f9406ae-2054-11eb-3f0c-1d03ba779f18
begin
	table_positions =
	[11, 5, 9, 4, 13, 14, 1, 15, 12, 6, 2, 16, 10, 17, 20, 3, 7, 8, 19, 18]

	games_won =
	[32, 25, 23, 21, 21, 19, 14, 13, 12, 12, 11, 11, 10, 11, 9, 9, 7, 8, 7, 6]

	teams_ = []
	for i in table_positions
		push!(teams_, teams[i])
	end

	table_position_df = DataFrame(Table_of_positions = teams_, Wins = games_won)

end

# ╔═╡ 099e4188-2054-11eb-1e7e-69bdb2b0202c
md"Let's now explore a little bit the a posteriori values we obtained."

# ╔═╡ 16572944-2045-11eb-38d0-fb20d981e3e9
begin
	post_att = collect(get(posterior, :att)[1])
	post_def = collect(get(posterior, :def)[1])
	post_home = collect(get(posterior, :home)[1])
end;

# ╔═╡ 4ae7d968-206f-11eb-1925-c5b5a31e6940
begin
using Plots
histogram(post_home, legend=false, title="Posterior distribution of home parameter")
end

# ╔═╡ 93d3bf98-5684-11eb-361e-c7b0f1a3bb0e
get(posterior, :att)[:att]

# ╔═╡ 18f36e1a-5682-11eb-2826-ab4d4c457af3
posterior.value.data

# ╔═╡ 6f096138-5682-11eb-179f-f973803b9268
collect(get(posterior, :att)[1])

# ╔═╡ 8cec9318-206e-11eb-3caa-3bad9d788d3c
md"As a first measure to analyze, it is interesting to see and quantify (if any) the effect that being local has on the score rate:"

# ╔═╡ 8b77d12c-206f-11eb-19a5-f557157ac05e
mean(post_home)

# ╔═╡ b56df7ea-206f-11eb-18b9-0544cc596ca5
md"So, to include in the model the parameter home was a good idea. indeed being local provides a very big advantage.

Beyond the fact that it is interesting to be able to quantify how much the location influences the scoring rate of the teams, including it in the analysis allow us to have better estimates of the defense and attack parameters of the teams. This is true because if it had not been included, this positive effect would have manifested itself in the only parameters it would have found, the attack and defense parameters, deforming the real measure of these.

So, being confident that we are on the right track, let´s find the attack and defence parameters of each team."

# ╔═╡ 554b4ef0-2045-11eb-3218-050ec936f1aa
begin
	teams_att = []
	teams_def = []
	for i in 1:length(post_att)
		push!(teams_att, post_att[i])
		push!(teams_def, post_def[i])
	end
end

# ╔═╡ 8a4df438-2045-11eb-2798-af592dbbffb5
md"This way we obtain all the samples of the posterior distributions for each one of the parameters of each equipment. Scroll right to explore the entire array."

# ╔═╡ 6b1a1520-2045-11eb-1302-0778b4e7a836
teams_att

# ╔═╡ da93bf52-2045-11eb-2b6f-3f31f57d8653
md"For example, if we would like to se the posterior distribution of the attack parameter for Burnley:"

# ╔═╡ 1fa92570-2046-11eb-3541-e5fccf7afabd
teams[1]

# ╔═╡ 255c4d08-2046-11eb-1cf0-b32d2cd711bf
histogram(teams_att[1], legend=false, title="Posterior distribution of Burnley´s attack power")

# ╔═╡ 473e6a46-205b-11eb-2731-7b47b99d86a5
mean(teams_att[1])

# ╔═╡ 917d4550-2058-11eb-174b-65436a4e6cc8
md"Comparing it to the attacking power of Manchester City, champion of the Premier league:"

# ╔═╡ c2de4714-2058-11eb-1a0d-f1bf96649844
teams[11]

# ╔═╡ 0103406e-2059-11eb-333f-7b5814f21a54
begin
	histogram(teams_att[11], legend=false, title="Posterior distribution of Manchester City´s attack power")
end

# ╔═╡ 579a4d9e-205b-11eb-2352-83beca145211
mean(teams_att[11])

# ╔═╡ 6924f372-2059-11eb-2b98-6f4b65b777f4
md"Okay, when comparing the league champion against a mid-table team, we can clearly see the superiority in attack. For now, it seems that the inference comes in handy.

Let's try now to have an overview of the attacking powers of each team. To do this, just take the average of each and plot it next to the standard deviation
"

# ╔═╡ ece1d052-2060-11eb-2b50-ef46663ed88f
begin
	teams_att_μ = mean.(teams_att)
	teams_def_μ = mean.(teams_def)
	teams_att_σ = std.(teams_att)
	teams_def_σ = std.(teams_def)
end;

# ╔═╡ 43a39b30-2061-11eb-28c5-37cbc4aed84b
md"""Remember that the "." operator is used for broadcasting. This means that it will apply the function to each component of the array"""

# ╔═╡ 41ca2316-2062-11eb-3fe6-f33a789fd578
begin
	teams_att_μ
	sorted_att = sortperm(teams_att_μ)
	abbr_names = [t[1:3] for t in teams]
end;

# ╔═╡ 0ef8a16e-2063-11eb-1391-6b3b959de541
begin
	abbr_names[5] = "Mun"
	abbr_names[10] = "Whu"
	abbr_names[11] = "Mci"
	abbr_names[16] = "Bou"
	abbr_names[18] = "Wba"
	abbr_names[19] = "Stk"
	abbr_names[20] = "Bha"
end;

# ╔═╡ 1eb73246-2063-11eb-0f20-416e433b5144
sorted_names = abbr_names[sorted_att]

# ╔═╡ abace764-2062-11eb-2949-fd9e86b50f17
begin
	scatter(1:20, teams_att_μ[sorted_att], grid=false, legend=false, yerror=teams_att_σ[sorted_att], color=:blue, title="Premier league 17/18 teams attack power")
	annotate!(collect(1:20), teams_att_μ[sorted_att] .+ 0.238, text.(sorted_names, :black, :center, 8))
	ylabel!("Mean team attack")
end

# ╔═╡ ebd267a4-2064-11eb-02a7-b93465de6319
md"""Although there is a high correlation between the attacking power of each team and its position on the table after the league ends, it is clear that this is not enough to explain the results. For example, Manchester City was the league's sub-champion, but only appeared in fifth place.

Let's explore what happens to the defence power: """

# ╔═╡ f6ad5f52-2065-11eb-0be7-f159941e0d89
begin
	sorted_def = sortperm(teams_def_μ)
	sorted_names_def = abbr_names[sorted_def]
end

# ╔═╡ 539e68b4-2066-11eb-09c2-b337241c36bc
begin
	scatter(1:20, teams_def_μ[sorted_def], grid=false, legend=false, yerror=teams_def_σ[sorted_def], color=:blue, title="Premier league 17/18 teams defence power")
	annotate!(collect(1:20), teams_def_μ[sorted_def] .+ 0.2, text.(sorted_names_def, :black, :center, 8))
	ylabel!("Mean team defence")
end

# ╔═╡ 69e7e54c-2067-11eb-21a1-cb72f4b423cd
md"To read this graph we have to remember that the defense effect is better the more negative it is, since it is representing the scoring rate that takes away from the opponent team. As we already said:"

# ╔═╡ f59d488a-2067-11eb-1d4b-1d9dd97ea39e
md"$θ_{team1} \sim att_{team1} + def_{team2}$"

# ╔═╡ 27d5a290-2068-11eb-06f5-0b16f6ec6cc5
md"As the $def_{team2}$ is adding up in the equation, if it take negative values, it is going to start substracting the scoring rate of the oponent.

Things, then, begin to make a little more sense. Now we can see that Manchester United is the team with the strongest defence, so being second in the overall is not extrange.

To gain a deeper understanding of what´s going on here, let's chart both characteristics together. This is going to let us see the combined effect they have. Also i´m going to add the final position of each team to improve the interpretability."

# ╔═╡ b8f9bae8-206a-11eb-1426-031f6fd05fd6
begin
	table_position =
	[11, 5, 9, 4, 13, 14, 1, 15, 12, 6, 2, 16, 10, 17, 20, 3, 7, 8, 19, 18]
	position = sortperm(table_position)
end

# ╔═╡ ee45d48e-206a-11eb-0edf-2b8b893bb583
begin
scatter(teams_att_μ, teams_def_μ, legend=false)
annotate!(teams_att_μ, teams_def_μ.+ 0.016, text.(abbr_names, :black, :center, 6))
annotate!(teams_att_μ, teams_def_μ.- 0.016, text.(position, :left, :center, 5))

xlabel!("Mean team attack")
ylabel!("Mean team defence")
end

# ╔═╡ c794f45e-206b-11eb-33c4-05bc429ba846
md"Well, great! Now we have some interesting information to analyze the teams and the league in general. It´s easier now to perceive how the two features interact with each other, comparing between teams and being able to see how that affects the final position.

For example, looking at the cases of Liverpool and Tottenham, or Leicester City and Everton; one could say (against general common sense) that the power of defense has a greater effect on the performance of each team than the attack. But we leave you to do those analysis for the betting house.

Well, we went from having a problem that seemed almost impossible to have a solid solution, with a quantitative analysis of the characteristics of each team. We even know how much the localization of the teams increases the scoring rate. We were able to achieve this thanks to the hierarchical framework that Bayesianism provides us. Using this tool allows us to create models proposing latent variables that cannot be observed, to infer them and to gain a much deeper and more generalized knowledge than we had at first. You just have to imagine a good story."

# ╔═╡ fcb7e8d2-2071-11eb-020d-7bb1d53f8a6d
md"### Bibliography

- [Paper of Gianluca Baio and Marta A. Blangiardo](https://discovery.ucl.ac.uk/id/eprint/16040/1/16040.pdf)
- [Post of Daniel Weitzenfeld](http://danielweitzenfeld.github.io/passtheroc/blog/2014/10/28/bayes-premier-league/)

"

# ╔═╡ fac413ec-5684-11eb-3167-d56d967247fc
mun_att_post = collect(get(posterior, :att)[:att])[5][:,1];

# ╔═╡ cc6681a2-5688-11eb-1d8e-e1edcb80a20d
mun_def_post = collect(get(posterior, :def)[:def])[5][:,1];

# ╔═╡ f01e7438-5688-11eb-2900-8df62e9feb18
liv_att_post = collect(get(posterior, :att)[:att])[4][:,1];

# ╔═╡ fae4e938-5688-11eb-3ab8-e3a8a4a61692
liv_def_post = collect(get(posterior, :def)[:def])[4][:,1];

# ╔═╡ 84d5e7a2-56dc-11eb-0d54-b790c7c714fb

# ╔═╡ 7704b9e6-56a0-11eb-0ade-2194a0845f61
post_home;

# ╔═╡ 03f063cc-5689-11eb-3d90-b7a5991332ba
# This function simulates matches given the attach, defense and home parameters.
# The first pair of parameters alwas correspond to the home team.
function simulate_matches_(att₁, def₁, att₂, def₂, home, n_matches, home_team=1)
	if home_team == 1
		logθ₁ = home + att₁ + def₂
		logθ₂ = att₂ + def₁

	elseif home_team == 2
		logθ₁ = att₁ + def₂
		logθ₂ = home + att₂ + def₁
	else
		return DomainError(home_team, "Invalid home_team value")
	end

	scores₁ = rand(LogPoisson(logθ₁), n_matches)
	scores₂ = rand(LogPoisson(logθ₂), n_matches)

	results = [(s₁, s₂) for (s₁, s₂) in zip(scores₁, scores₂)]

	return results
end

# ╔═╡ a8a9179c-5693-11eb-1758-e7d34722c7fa
mat = simulate_matches_(0.2, -0.1, 0.4, -0.3, 0.2, 1000, 2)

# ╔═╡ 7e82f15c-5695-11eb-20f1-85763cbfa572
max_h = maximum(map(x -> x[1], mat))

# ╔═╡ bd11306c-5697-11eb-22ed-99ae7703be59
max_a = maximum(map(x -> x[2], mat))

# ╔═╡ d99e52fa-5697-11eb-12fe-ab4ae36ab362
function simulate_matches(team1_att_post, team1_def_post, team2_att_post,				team2_def_post, home_post, n_matches)

	team1_as_home_results = Tuple{Int64,Int64}[]
	team2_as_home_results = Tuple{Int64,Int64}[]

	for (t1_att, t1_def, t2_att, t2_def, home) in zip(team1_att_post, team1_def_post, team2_att_post, team2_def_post, home_post)

		team1_as_home_results = vcat(team1_as_home_results, simulate_matches_(t1_att, t1_def, t2_att, t2_def, home, n_matches, 1))

		team2_as_home_results = vcat(team2_as_home_results, simulate_matches_(t1_att, t1_def, t2_att, t2_def, home, n_matches, 2))
	end

	return team1_as_home_results, team2_as_home_results
end

# ╔═╡ 5a3d6966-56a0-11eb-0db1-c5a06ea2dd83
kjj, kjjj = simulate_matches(mun_att_post, mun_def_post, liv_att_post, liv_def_post, post_home, 1000)

# ╔═╡ 9d1f1e7e-56a1-11eb-2539-f5da33710a32
function match_heatmaps(team1_as_home_results, team2_as_home_results)

	max_t1_as_home = maximum(map(x -> x[1], team1_as_home_results))
	max_t2_as_away = maximum(map(x -> x[2], team1_as_home_results))

	max_t1_as_away = maximum(map(x -> x[1], team2_as_home_results))
	max_t2_as_home = maximum(map(x -> x[2], team2_as_home_results))

	matrix_t1_as_home = zeros(Int64, (max_t1_as_home + 1, max_t2_as_away + 1))
	matrix_t2_as_home = zeros(Int64, (max_t1_as_away + 1, max_t2_as_home + 1))

	for match in team1_as_home_results
		matrix_t1_as_home[match[1] + 1, match[2] + 1] += 1
	end

	for match in team2_as_home_results
		matrix_t2_as_home[match[1] + 1, match[2] + 1] += 1
	end

	gr()
	#heat_t1_home = heatmap(0:max_t1_as_home,
	heatmap(0:max_t1_as_home,
								0:max_t2_as_away,
								matrix_t1_as_home,
								c=cgrad([:blue, :white,:red, :yellow]),
								xlabel="Team1 score", ylabel="Team2 score",
								title="Team1 as home")
	savefig("./t1_as_home")

	#heat_t2_home = heatmap(0:max_t1_as_away,
	heatmap(0:max_t1_as_away,
								0:max_t2_as_home,
								matrix_t2_as_home,
								c=cgrad([:blue, :white,:red, :yellow]),
								xlabel="Team1 Score", ylabel="Team2 Score",
								title="Team2 as home")
	savefig("./t2_as_home")
	#plot(heat_t1_home, heat_t2_home, layout=(2,1), size=(900, 400))
	#current()

	return matrix_t1_as_home, matrix_t2_as_home
end

# ╔═╡ d7bfc106-56d0-11eb-21b3-3b9036905041
m_1_home, m_t2_home = match_heatmaps(kjj, kjjj)

# ╔═╡ 9ceba0fa-56d4-11eb-2696-21f4babf699c
begin
	gr()
	heatmap(collect(0:(size(m_t2_home, 1)-1)),
			collect(0:(size(m_t2_home, 2)-1)),
			m_t2_home,
			c=cgrad([:blue, :white,:red, :yellow]),
			xlabel="Team1 score", ylabel="Team2 score",
			title="Team1 as home")
end

# ╔═╡ 0da9d3a0-56d5-11eb-08a9-c9e83106dfc3
m_t1_home

# ╔═╡ db8ed140-5735-11eb-2078-2f90d15d6325
"hola"

# ╔═╡ Cell order:
# ╟─2414104e-1d41-11eb-37cb-9b5cee78a98b
# ╟─3b7f8a1e-1d42-11eb-1ab1-c11fda416e80
# ╟─4e4385b4-203b-11eb-0c5b-cbe41156401b
# ╟─2dca2f42-203b-11eb-2cb3-0d6a3823cdcd
# ╠═a4f0a914-1ec9-11eb-3c22-db87a9776aaa
# ╠═0c0268dc-1ed3-11eb-2f73-b3706305b298
# ╠═cbaaf72c-1eb4-11eb-3509-932b337f270b
# ╠═ab6fcb22-1ed2-11eb-2749-3bef16911972
# ╠═d2a81a14-1ed2-11eb-1307-bfb169aebbb4
# ╠═dd80fc38-1ed2-11eb-0e8c-49859d27c72c
# ╠═7f0c72d4-1f9e-11eb-06f1-cb587d7e5436
# ╟─868e5c8c-1ed3-11eb-291c-fb26d512c103
# ╟─04a8f16c-1ed5-11eb-3979-af35c6d9dea1
# ╟─9f1e1618-1edb-11eb-02af-9fa5c8904163
# ╟─42104900-1f80-11eb-019f-a7c561ca3e4b
# ╟─9d0f3a1e-1f80-11eb-3422-fb7b4696cc13
# ╟─0b1f9c68-1f83-11eb-11e7-e7614c021c05
# ╟─043c438c-1f84-11eb-2cd7-6dd07a125910
# ╟─4a31541e-1f97-11eb-038e-736446224c21
# ╟─e3a99868-1f97-11eb-1872-79c8f6abe1be
# ╟─c3ca1cc4-1f98-11eb-3c23-33acb95fbc03
# ╟─4106e55a-1f99-11eb-110e-b79221774f12
# ╟─bff4fdae-1f99-11eb-2d94-aba7bf9d097f
# ╟─2b135202-1fa7-11eb-324d-93a5eda41e3e
# ╟─ddc7b166-203a-11eb-0ac2-056c309bb590
# ╟─8e164be0-203b-11eb-293a-6b32dba96133
# ╠═5f83510e-1fc5-11eb-3c40-83a844bf2907
# ╠═51a309bc-2033-11eb-10c0-ed17545df33d
# ╟─f3f0b6f8-2033-11eb-0f9e-d951d791001d
# ╠═f3f4bfba-203f-11eb-142c-a187112744d2
# ╠═2ce4435c-2040-11eb-1670-e39ad4cc690c
# ╠═9e50e56e-5674-11eb-2919-4bde8964bc59
# ╟─8c1dd9de-2040-11eb-35a3-39e8196577a1
# ╟─0f9406ae-2054-11eb-3f0c-1d03ba779f18
# ╟─099e4188-2054-11eb-1e7e-69bdb2b0202c
# ╠═16572944-2045-11eb-38d0-fb20d981e3e9
# ╠═93d3bf98-5684-11eb-361e-c7b0f1a3bb0e
# ╠═18f36e1a-5682-11eb-2826-ab4d4c457af3
# ╠═6f096138-5682-11eb-179f-f973803b9268
# ╟─8cec9318-206e-11eb-3caa-3bad9d788d3c
# ╠═4ae7d968-206f-11eb-1925-c5b5a31e6940
# ╠═8b77d12c-206f-11eb-19a5-f557157ac05e
# ╟─b56df7ea-206f-11eb-18b9-0544cc596ca5
# ╠═554b4ef0-2045-11eb-3218-050ec936f1aa
# ╟─8a4df438-2045-11eb-2798-af592dbbffb5
# ╠═6b1a1520-2045-11eb-1302-0778b4e7a836
# ╟─da93bf52-2045-11eb-2b6f-3f31f57d8653
# ╠═1fa92570-2046-11eb-3541-e5fccf7afabd
# ╠═255c4d08-2046-11eb-1cf0-b32d2cd711bf
# ╠═473e6a46-205b-11eb-2731-7b47b99d86a5
# ╟─917d4550-2058-11eb-174b-65436a4e6cc8
# ╠═c2de4714-2058-11eb-1a0d-f1bf96649844
# ╠═0103406e-2059-11eb-333f-7b5814f21a54
# ╠═579a4d9e-205b-11eb-2352-83beca145211
# ╟─6924f372-2059-11eb-2b98-6f4b65b777f4
# ╠═ece1d052-2060-11eb-2b50-ef46663ed88f
# ╟─43a39b30-2061-11eb-28c5-37cbc4aed84b
# ╠═41ca2316-2062-11eb-3fe6-f33a789fd578
# ╟─0ef8a16e-2063-11eb-1391-6b3b959de541
# ╠═1eb73246-2063-11eb-0f20-416e433b5144
# ╠═abace764-2062-11eb-2949-fd9e86b50f17
# ╟─ebd267a4-2064-11eb-02a7-b93465de6319
# ╠═f6ad5f52-2065-11eb-0be7-f159941e0d89
# ╠═539e68b4-2066-11eb-09c2-b337241c36bc
# ╟─69e7e54c-2067-11eb-21a1-cb72f4b423cd
# ╟─f59d488a-2067-11eb-1d4b-1d9dd97ea39e
# ╟─27d5a290-2068-11eb-06f5-0b16f6ec6cc5
# ╠═b8f9bae8-206a-11eb-1426-031f6fd05fd6
# ╠═ee45d48e-206a-11eb-0edf-2b8b893bb583
# ╟─c794f45e-206b-11eb-33c4-05bc429ba846
# ╟─fcb7e8d2-2071-11eb-020d-7bb1d53f8a6d
# ╠═fac413ec-5684-11eb-3167-d56d967247fc
# ╠═cc6681a2-5688-11eb-1d8e-e1edcb80a20d
# ╠═f01e7438-5688-11eb-2900-8df62e9feb18
# ╠═fae4e938-5688-11eb-3ab8-e3a8a4a61692
# ╠═84d5e7a2-56dc-11eb-0d54-b790c7c714fb
# ╠═7704b9e6-56a0-11eb-0ade-2194a0845f61
# ╠═03f063cc-5689-11eb-3d90-b7a5991332ba
# ╠═a8a9179c-5693-11eb-1758-e7d34722c7fa
# ╠═7e82f15c-5695-11eb-20f1-85763cbfa572
# ╠═bd11306c-5697-11eb-22ed-99ae7703be59
# ╠═d99e52fa-5697-11eb-12fe-ab4ae36ab362
# ╠═5a3d6966-56a0-11eb-0db1-c5a06ea2dd83
# ╠═9d1f1e7e-56a1-11eb-2539-f5da33710a32
# ╠═d7bfc106-56d0-11eb-21b3-3b9036905041
# ╠═9ceba0fa-56d4-11eb-2696-21f4babf699c
# ╠═0da9d3a0-56d5-11eb-08a9-c9e83106dfc3
# ╠═db8ed140-5735-11eb-2078-2f90d15d6325


f(x) = x + 100

macro g()         # According to Julia docs, ...
  :(f(x) + 5)     # ... f is global, x is local, right?
end               # if so, f should refer to the f above?

(function main()
  local x = 3
  f(x) = x - 100  # f in the call environment subtracts 100
#  println(@g())   # So why does this do -92?
end)()


"""
    nullgraph!(net)

Remove all nodes and links from the network.

The first argument *net* is a FastNet structure that is be used in the simulation.

# Examples
```jldoctest
julia> using Fastnet

julia> net=FastNet(1000,2000,1,[])
Network of 0 nodes and 0 links

julia> randomgraph!(net)
Network of 1000 nodes and 2000 links

julia> nullgraph!(net)
Network of 0 nodes and 0 links
```
"""
function nullgraph!(net::FastNet)
    while countnodes_f(net)>0
        destroynode_f!(net,net.nid[1])
    end
    net
end

"""
    randomgraph!(net;<keyword arguments>)

Create an ER random graph in the network *net*.

The network isn't guaranteed to be a simple graph, but in large sparse
networks it is simple with high probability.

By default all nodes and links that the network can accommodate will be used and
all nodes will be set to state one. This behavior can be controlled by the following
keyword arguments:

- N : The number of nodes that will be used in the creation of the random graph.
  All other nodes will be removed from the network.
- K : The number of links that will be used in the creation of the random graph.
  All other links will be removed from the network.
- S : The state of the nodes. All nodes will be set to this state.

# Examples
```jldoctest
julia> using Fastnet

julia> net=FastNet(1000,2000,1,[])
Network of 0 nodes and 0 links

julia> randomgraph!(net)
Network of 1000 nodes and 2000 links

julia> nullgraph!(net)
Network of 0 nodes and 0 links

julia> randomgraph!(net,N=100,K=10)
Network of 100 nodes and 10 links
```
"""
function randomgraph!(net::FastNet; N::Int=0,K::Int=0,S::Int=1)
    nullgraph!(net)
    n=N;
    k=K;
    s=S;
    if n===0
        n=net.N
    end
    if k===0
        k=net.K
    end
    if n<1 && k>0
        throw(ArgumentError("In order to create links the net has to have at least one node"))
    end
    if n>net.N
        throw(ArgumentError("Trying to create more nodes than maximum allowed by the net"))
    end
    if k>net.K
        throw(ArgumentError("Trying to create more links than maximum allowed by the net"))
    end
    if s<1 || s>net.C-1
        msg="The net passed to randomgraph! only supports node states between 1 and $(net.C-1),"
        msg*=" but you are asking it to set nodes to state $s."
        throw(ArgumentError(msg))
    end
    makenodes!(net,n,s)
    for i=1:k
        src=randomnode_f(net,s)
        dst=randomnode_f(net,s)
        makelink_f!(net,src,dst)
    end
    net
end

"""
    configmodel!(net::FastNet,degreedist;<keyword arguments>)

Create a configuration model style network with prescribed degree distribution, *degreedist*.

The network in *net* is replaced with the new topology. The degree distribution is specified in
terms of a vector of Float64 variables such that *degreedist[k]* is specifies, pk, the probability
that a randomly drawn node has degree k. If the elements of *degreedist* add up to less than 1.0 the
remaining nodes will have degree zero.

The network generation is fast and unbiased but isn't guaranteed to result in a simple graph.
The algorithm will try to match the desired degree distribution as closely as possible,
but small discrepancies can appear if the degree distribution would result in an odd degree sum
or non integer numbers of nodes of certain degrees.

If there FastNet is not large enough to accomodate the desired number of links or nodes an argument error
will be thrown.

The keyword arguments are
- N : The number of nodes that will be used in the creation of the network
- S : The state of the nodes. All nodes will be set to this state.

# Examples
```jldoctest
julia> using Fastnet

julia> net=FastNet(1000,2000,2,[])
Network of 0 nodes and 0 links

julia> configmodel!(net,[0.5,0.25,0.25],N=200)
Network of 200 nodes and 175 links

julia> degreedist(net)
3-element Vector{Float64}:
 0.5
 0.25
 0.25

julia> configmodel!(net,[0.5,0.25],N=200)
Network of 200 nodes and 100 links

julia> degreedist(net)
2-element Vector{Float64}:
 0.5
 0.25
```
"""
function configmodel!(net::FastNet,degreedist;N=0,S=1)
    (counts,totalstubs)=_configmodelsetup!(net,degreedist,N,S)
    nn=sum(counts);
    l=length(counts)
    nde=Array{Int,1}(undef,totalstubs)
    curnode=1
    curstub=1
    for i=1:l
        for j=1:counts[i]
            for s=1:i
                nde[curstub]=curnode
                curstub+=1
            end
            curnode+=1
        end
    end
    cutoff=totalstubs
    rng=net.rng
    while cutoff>1
        stub=rand(rng,1:cutoff)
        src=node_f(net,nde[stub])
        nde[stub]=nde[cutoff]
        cutoff-=1
        stub=rand(rng,1:cutoff)
        dst=node_f(net,nde[stub])
        nde[stub]=nde[cutoff]
        cutoff-=1
        makelink_f!(net,src,dst)
    end
    net
end

"""
    regulargraph!(net::FastNet,deg;<keyword arguments>)

Create a regular graph with node degree *deg*.

The network in *net* is replaced with the new topology in which all nodes have degree *deg*
and are randomly connected. If the number of nodes is not specified the function will
try to use all nodes allowed by net.

The network generation is fast and unbiased, but isn't guaranteed to result in a simple graph.

Note that finite regular graphs with odd node degree and odd number of nodes do not exist. Hence
either *deg* or the number of nodes must be even.

If there FastNet is not large enough to accomodate the desired number of links or nodes an argument error
will be thrown.

The keyword arguments are
- N : The number of nodes that will be used in the creation of the network
- S : The state of the nodes. All nodes will be set to this state.

# Examples
```jldoctest
julia> using Fastnet

julia>  net=FastNet(1000,2000,2,[])
Network of 0 nodes and 0 links

julia> regulargraph!(net,4)
Network of 1000 nodes and 2000 links

julia> degreedist(net)
4-element Vector{Float64}:
 0.0
 0.0
 0.0
 1.0
```
"""
function regulargraph!(net::FastNet,deg;N=0,S=1)
    s=0
    n=0
    d=0
    try
        s=convert(Int,S)
    catch e
        throw(ArgumentError("regulargraph expects s to be an integer"))
    end
    try
        n=convert(Int,N)
    catch e
        throw(ArgumentError("regulargraph expects N to be an integer"))
    end
    try
        d=convert(Int,deg)
    catch e
        throw(ArgumentError("regulargraph expects deg to be an integer"))
    end
    if n==0
        n=net.N
    end
    checknodestate(net,s,"Trying to create regular graph")
    if n<1
        throw(ArgumentError("regulargraph expects number of nodes n to be positive"))
    end
    if d<0
        throw(ArgumentError("regulargraph expects nodedegree deg to be non-negative"))
    end
    if n>net.N
        throw(ArgumentError("Requested size for regulargraph exceeds max node count of underlyng FastNet"))
    end
    if (n*d)÷2>net.K
        throw(ArgumentError("Requested regulargraph exceeds max link count of underlyng FastNet"))
    end
    if isodd(n) && isodd(d)
        throw(ArgumentError("A regular graph of odd node degree must have an even number of nodes."))
    end
    nullgraph!(net)
    makenodes!(net,n,s)
    totalstubs=n*d
    nde=Array{Int,1}(undef,totalstubs)
    for i=1:totalstubs
        nde[i]=((i-1)÷deg)+1
    end
    cutoff=totalstubs
    rng=net.rng
    while cutoff>0
        srcs=rand(rng,1:cutoff)         # pick the source
        src=node_f(net,nde[srcs])
        nde[srcs]=nde[cutoff]
        cutoff-=1
        dsts=rand(rng,1:cutoff)         # pick the source
        dst=node_f(net,nde[dsts])
        nde[dsts]=nde[cutoff]
        cutoff-=1
        makelink_f!(net,src,dst)
    end
    net
end

"""
    rectlattice!(net::FastNet,dims, <keyword arguments>)

Create a rectangular lattice with given dimensions *dims*.

The network in *net* is replaced with the new topology, that is a lattice specified by dims.
*dims* can be a number, in this case it indicates the number of nodes to be arranged into a 1D lattice.
Alternatively, *dims* can be a vector of Ints. In this case the dimension of the lattice is identical to the
length of *dims* and each element of *dims* specifies the length of the lattice in one of these dimensions.

If there FastNet is not large enough to accomodate the desired number of nodes or links an argument error
will be thrown.

Keyword arguments are
- periodic : If this argument is true the lattice is generated with periodic boundary conditions in all dimensions.
  Alternatively a Vector of Bool of the same length as *dims* can be supplied. In this case the n'th argument of
  the vector specifies if the lattice is periodic in the n'th dimension.
- S : The state of the nodes. All nodes will be set to this state.

# Examples
```jldoctest
julia> using Fastnet

julia>  net=FastNet(2000,6000,2,[])
Network of 0 nodes and 0 links

julia> rectlattice!(net,[10,20,10],periodic=[true,false,true])
Network of 2000 nodes and 5900 links

julia> degreedist(net)
6-element Vector{Float64}:
 0.0
 0.0
 0.0
 0.0
 0.1
 0.9
```
"""
function rectlattice!(
        net         ::FastNet,
        dims        ::Union{Int,Tuple,AbstractVector};
        S           ::Integer=1,
        periodic    ::Union{AbstractArray,Bool,Tuple}=false
    )
    task="Trying to create a rectangular lattice"
    d=[dims...]
    nd=length(d)
    per=[false]
    if isa(periodic,Bool)
        per=fill(periodic,nd)
    else
        per=[periodic...]
    end
    if length(per)!=length(d)
        throw(ArgumentError(task*", but the number of values passed for parameter periodic does not agree with dims"))
    end
    for x in per
        if !isa(x,Bool)
            throw(ArgumentError(task*", but $x in periodic is not of type Bool"))
        end
    end
    n=1
    s=checknodestate(net,S,task)
    for i=1:nd
        n*=d[i]
    end
    nullgraph!(net)
    makenodes!(net,n,s)
    linksneeded=0
    for i=1:nd
        linksneeded+=n
        if !per[i]
            linksneeded-=n÷d[i]
        end
    end
    for i=1:n
        lowmult=1
        for j=1:nd
            if ((i-1)÷lowmult)%d[j]==d[j]-1
                if per[j]
                    src=node_f(net,i)
                    dst=node_f(net,i+lowmult-d[j]*lowmult)
                    makelink_f!(net,src,dst)
                end
            else
                src=node_f(net,i)
                dst=node_f(net,i+lowmult)
                makelink_f!(net,src,dst)
            end
            lowmult*=d[j]
        end
    end
    net
end

"""
    adjacency!(net,mat;S=1)

Create a network with given adjacency matrix.

The network in *net* is replaced with the new topology that is specified by the adjacency matrix *mat*.
If direction of links matters note that the element *mat[i,j]* corresponds to the link from j to i.

Symmeric matrices will not result in parallel links, instead the link is placed in an arbitrary direction.

Note that node *n* in the matrix will be the node in position *n* in *net* after creation, which is
not necessarily the node with ID *n*, if you need to find a particular node at a later time then it
is best to save its id using the node(net,pos) function directly after calling adjacency!(net,mat).

If *net* is not large enough to accomodate the desired number of nodes or links an argument error
will be thrown.

Keyword arguments are
- S : The state of the nodes. All nodes will be set to this state.

# Examples
```jldoctest
julia> using Fastnet

julia> net=FastNet(1000,2000,2,[])
Network of 0 nodes and 0 links

julia> mat=[0 1 0; 1 0 1; 0 1 0]
3×3 Matrix{Int64}:
 0  1  0
 1  0  1
 0  1  0

julia> adjacency!(net,mat)
Network of 3 nodes and 2 links
```
"""
function adjacency!(
        net     ::FastNet,
        mat     ::AbstractMatrix;
        S       ::Integer=1
    )
    task="Trying to create topology from adjacency matrix"
    x,y=size(mat)
    if x!=y
        err=task*", but adjacency! expects a square matrix as its second argument and received a rectangular one"
        throw(ArgumentError(err))
    end
    if x>net.N
        throw(ArgumentError(task*", but the matrix is larger than the maximum number of allowed nodes by the network"))
    end
    s=checknodestate(net,S,task)
    count=0
    b1=false
    b2=false
    for i=1:x-1
        for j=i+1:x
            try
                b1=convert(Bool,mat[i,j])
                b2=convert(Bool,mat[i,j])
            catch e
                throw(ArgumentError(task*", but was unable to convert an element of mat to Bool"))
            end
            if b1||b2
                count+=1
            end
        end
    end
    if count>net.K
        throw(ArgumentError(task*", but the matrix contains more links than permitted by net"))
    end
    nullgraph!(net)
    makenodes!(net,x,s)
    for i=1:x
        dst=node_f(net,i)
        for j=i+1:x
            src=node_f(net,j)
            linked::Bool = mat[i,j]
            if linked
                makelink_f!(net,src,dst)
            else
                linked = mat[j,i]
                if linked
                    makelink_f!(net,dst,src)
                end
            end
        end
    end
    net
end

### WIP
function configmodel_DG!(net::FastNet,degreedist,N::Int=0,S::Int=1)
    (counts,totalstubs)=_configmodelsetup!(net,degreedist,N,S)

    println("On track")
end

"""
    randomgeometricgraph!(net::FastNet; N::Int,K::Int,S::Int, dim::Int, deg::Float64)

Create a random geometric graph with mean degree *deg*.

The network in *net* is replaced with the new topology in which each node has *dim* vector entries uniformly drawn
from (0,1). A maximal connection distance is calculated from the given mean degree and nodes get connected,
when their euclidean distance is smaller than the maximal connection distance.

If there FastNet is not large enough to accomodate the desired number of links or nodes an argument error
will be thrown.

Additionaly, if the calculated maximal connection distance results in more links being generated than supported by the net,
an argument error will be thrown as well.

The keyword arguments are
- N : The number of nodes that will be used in the creation of the network
- S : The state of the nodes. All nodes will be set to this state.
- dim : Dimensions of the random geometric graph.
- deg : The graph´s mean degree which is used to calculate the maximal connection distance between nodes.

# Examples
```jldoctest
julia> using Fastnet

julia>  net=FastNet(1000,2000,2,[])
Network of 0 nodes and 0 links

julia> randomgeometricgraph!(net,dim=2,meandegree=2.0)
Network of 1000 nodes and 970 links

```
"""
function randomgeometricgraph!(net::FastNet; N::Int=0,K::Int=0,S::Int=1, dim::Int=2, deg::Float64=2.0)
    nullgraph!(net)
    n=N;
    k=K;
    s=S;
    if n===0
        n=net.N
    end
    if k===0
        k=net.K
    end
    if n<1 && k>0
        throw(ArgumentError("In order to create links the net has to have at least one node"))
    end
    if n>net.N
        throw(ArgumentError("Trying to create more nodes than maximum allowed by the net"))
    end
    if k>net.K
        throw(ArgumentError("Trying to create more links than maximum allowed by the net"))
    end
    if s<1 || s>net.C-1
        msg="The net passed to randomgeometricgraph! only supports node states between 1 and $(net.C-1),"
        msg*=" but you are asking it to set nodes to state $s."
        throw(ArgumentError(msg))
    end

    r = (1/2)*((deg/n) * double_factorial(dim) / (pi/2)^(floor(dim/2)))^(1/dim)
    makenodes!(net,n,s)
    loc = rand(n,dim)
    for i = 1:n
        for j = i+1:n
            dist = sqrt(sum((loc[i,:].-loc[j,:]).^2))
            if (dist < r)
                if countlinks_f(net)>=k
                    throw(ArgumentError("For the given mean degree, you would create more links than allowed by the net"))
                end
                makelink_f!(net,i,j)
            end
        end
    end
    net
end

function double_factorial(n::Int)
    if isodd(n)
        k=Int((n+1)/2)
        return factorial(2*k-1)/(2^(k-1)*factorial(k-1))
    else
        k=Int(n/2)
        return 2^k*factorial(k)
    end
end


# This file contains SDESystem prototypes

import DifferentialEquations: LambaEM, SDEProblem
import UUIDs: uuid4

"""
    @def_sde_system ex

where `ex` is the expression to define to define a new AbstractSDESystem component type. The usage is as follows:
```julia
@def_sde_system mutable struct MySDESystem{T1,T2,T3,...,TN,OP,RH,RO,ST,IP,OP} <: AbstractSDESystem
    param1::T1 = param1_default                 # optional field
    param2::T2 = param2_default                 # optional field
    param3::T3 = param3_default                 # optional field
        ⋮
    paramN::TN = paramN_default                 # optional field
    drift::DR = drift_function                  # mandatory field
    diffusion::DF = diffusion_function          # mandatory field
    readout::RO = readout_functtion             # mandatory field
    state::ST = state_default                   # mandatory field
    input::IP = input_default                   # mandatory field
    output::OP = output_default                 # mandatory field
end
```
Here, `MySDESystem` has `N` parameters. `MySDESystem` is represented by the `drift`, `diffusion` and `readout` function. `state`, `input` and `output` is the initial state, input port and output port of `MySDESystem`.

!!! warning
    `drift` must have the signature
    ```julia
    function drift((dx, x, u, t, args...; kwargs...)
        dx .= .... # update dx
    end
    ```
    and `diffusion` must have the signature
    ```julia
    function diffusion((dx, x, u, t, args...; kwargs...)
        dx .= .... # update dx
    end
    ```
    and `readout` must have the signature
    ```julia
    function readout(x, u, t)
        y = ...
        return y
    end
    ```

!!! warning
    New SDE system must be a subtype of `AbstractSDESystem` to function properly.

# Example
```julia
julia> @def_sde_system mutable struct MySDESystem{DR, DF, RO, IP, OP} <: AbstractSDESystem
           η::Float64 = 1.
           drift::DR = (dx, x, u, t) -> (dx .= x)
           diffusion::DF = (dx, x, u, t, η=η) -> (dx .= η)
           readout::RO = (x, u, t) -> x
           state::Vector{Float64} = rand(2)
           input::IP = nothing
           output::OP = Outport(2)
       end

julia> ds = MySDESystem();
```
"""
macro def_sde_system(ex)
    checksyntax(ex, :AbstractSDESystem)
    appendcommonex!(ex)
    foreach(nex -> appendex!(ex, nex), [
        :( alg::$ALG_TYPE_SYMBOL = Causal.LambaEM{true}() ),
        :( integrator::$INTEGRATOR_TYPE_SYMBOL = Causal.construct_integrator(Causal.SDEProblem, input,
            (drift, diffusion), state, t, modelargs, solverargs; alg=alg, modelkwargs=modelkwargs, solverkwargs=solverkwargs, numtaps=3) )
        ])
    quote
        Base.@kwdef $ex
    end |> esc
end

##### Define SDE system library

"""
    SDESystem(; drift, diffusion, readout, state, input, output)

Constructs a SDE system.
"""
@def_sde_system mutable struct SDESystem{DR, DF, RO, ST, IP, OP} <: AbstractSDESystem
    drift::DR
    diffusion::DF
    readout::RO
    state::ST
    input::IP
    output::OP
end

@doc raw"""
    NoisyLorenzSystem()

Constructs a noisy Lorenz system
"""
@def_sde_system mutable struct NoisyLorenzSystem{ET, DR, DF, RO, IP, OP} <: AbstractSDESystem
    σ::Float64 = 10.
    β::Float64 = 8 / 3
    ρ::Float64 = 28.
    η::ET = 1.
    γ::Float64 = 1.
    drift::DR = function lorenzdrift(dx, x, u, t, σ=σ, β=β, ρ=ρ, γ=γ)
        dx[1] = σ * (x[2] - x[1])
        dx[2] = x[1] * (ρ - x[3]) - x[2]
        dx[3] = x[1] * x[2] - β * x[3]
        dx .*= γ
    end
    diffusion::DF = (dx, x, u, t, η=η) -> (dx .= η)
    readout::RO = (x, u, t) -> x
    state::Vector{Float64} = rand(3)
    input::IP = nothing
    output::OP = Outport(3)
end

@doc raw"""
    NoisyLorenzSystem()

Constructs a noisy Lorenz system
"""
@def_sde_system mutable struct ForcedNoisyLorenzSystem{ET, CM, DR, DF, RO, IP, OP} <: AbstractSDESystem
    σ::Float64 = 10.
    β::Float64 = 8 / 3
    ρ::Float64 = 28.
    η::ET = 1.
    cplmat::CM = I(3)
    γ::Float64 = 1.
    drift::DR = function forcedlorenzdrift(dx, x, u, t, σ=σ, β=β, ρ=ρ, γ=γ, cplmat=cplmat)
        dx[1] = σ * (x[2] - x[1])
        dx[2] = x[1] * (ρ - x[3]) - x[2]
        dx[3] = x[1] * x[2] - β * x[3]
        dx .*= γ
        dx .+= cplmat * map(ui -> ui(t), u.itp)   # Couple inputs
    end
    diffusion::DF = (dx, x, u, t, η=η) -> (dx .= η)
    readout::RO = (x, u, t) -> x
    state::Vector{Float64} = rand(3)
    input::IP = Inport(3)
    output::OP = Outport(3)
end

##### Pretty printing
show(io::IO, ds::SDESystem) = print(io,
    "SDESystem(drift:$(ds.drift), diffusion:$(ds.diffusion), readout:$(ds.readout), state:$(ds.state), t:$(ds.t), ",
    "input:$(ds.input), output:$(ds.output))")
show(io::IO, ds::NoisyLorenzSystem) = print(io,
    "NoisyLorenzSystem(σ:$(ds.σ), β:$(ds.β), ρ:$(ds.ρ), η:$(ds.η), γ:$(ds.γ), state:$(ds.state), t:$(ds.t), ",
    "input:$(ds.input), output:$(ds.output))")
show(io::IO, ds::ForcedNoisyLorenzSystem) = print(io,
    "ForcedNoisyLorenzSystem(σ:$(ds.σ), β:$(ds.β), ρ:$(ds.ρ), η:$(ds.η), γ:$(ds.γ), cplmat:$(ds.cplmat), ",
    "state:$(ds.state), t:$(ds.t), input:$(ds.input), output:$(ds.output))")