## RESAMPLING STRATEGIES

abstract type ResamplingStrategy <: MLJType end
show_as_constructed(::Type{<:ResamplingStrategy}) = true

# resampling strategies are `==` if they have the same type and their
# field values are `==`:
function ==(s1::S, s2::S) where S <: ResamplingStrategy
    return all(getfield(s1, fld) == getfield(s2, fld) for fld in fieldnames(S))
end

# fallbacks:
train_test_pairs(s::ResamplingStrategy, rows, X, y, w) =
    train_test_pairs(s, rows, X, y)
train_test_pairs(s::ResamplingStrategy, rows, X, y) =
    train_test_pairs(s, rows, y)
train_test_pairs(s::ResamplingStrategy, rows, y) =
    train_test_pairs(s, rows)

# Helper to interpret rng, shuffle in case either is `nothing` or if
# `rng` is an integer:
function shuffle_and_rng(shuffle, rng)
    if rng isa Integer
        rng = MersenneTwister(rng)
    end

    if shuffle === nothing
        shuffle = ifelse(rng===nothing, false, true)
    end

    if rng === nothing
        rng = Random.GLOBAL_RNG
    end

    return shuffle, rng
end

"""
    holdout = Holdout(; fraction_train=0.7,
                         shuffle=nothing,
                         rng=nothing)

Holdout resampling strategy, for use in `evaluate!`, `evaluate` and in tuning.

    train_test_pairs(holdout, rows)

Returns the pair `[(train, test)]`, where `train` and `test` are
vectors such that `rows=vcat(train, test)` and
`length(train)/length(rows)` is approximatey equal to fraction_train`.

Pre-shuffling of `rows` is controlled by `rng` and `shuffle`. If `rng`
is an integer, then the `Holdout` keyword constructor resets it to
`MersenneTwister(rng)`. Otherwise some `AbstractRNG` object is
expected.

If `rng` is left unspecified, `rng` is reset to `Random.GLOBAL_RNG`,
in which case rows are only pre-shuffled if `shuffle=true` is
specified.

"""
struct Holdout <: ResamplingStrategy
    fraction_train::Float64
    shuffle::Bool
    rng::Union{Int,AbstractRNG}

    function Holdout(fraction_train, shuffle, rng)
        0 < fraction_train < 1 ||
            error("`fraction_train` must be between 0 and 1.")
        return new(fraction_train, shuffle, rng)
    end
end

# Keyword Constructor
Holdout(; fraction_train::Float64=0.7, shuffle=nothing, rng=nothing) =
    Holdout(fraction_train, shuffle_and_rng(shuffle, rng)...)

function train_test_pairs(holdout::Holdout, rows)

    train, test = partition(rows, holdout.fraction_train,
                          shuffle=holdout.shuffle, rng=holdout.rng)
    return [(train, test),]

end

"""
    cv = CV(; nfolds=6,  shuffle=nothing, rng=nothing)

Cross-validation resampling strategy, for use in `evaluate!`,
`evaluate` and tuning.

    train_test_pairs(cv, rows)

Returns an `nfolds`-length iterator of `(train, test)` pairs of
vectors (row indices), where each `train` and `test` is a sub-vector
of `rows`. The `test` vectors are mutually exclusive and exhaust
`rows`. Each `train` vector is the complement of the corresponding
`test` vector. With no row pre-shuffling, the order of `rows` is
preserved, in the sense that `rows` coincides precisely with the
concatenation of the `test` vectors, in the order they are
generated. All but the last `test` vector have equal length.

Pre-shuffling of `rows` is controlled by `rng` and `shuffle`. If `rng`
is an integer, then the `CV` keyword constructor resets it to
`MersenneTwister(rng)`. Otherwise some `AbstractRNG` object is
expected.

If `rng` is left unspecified, `rng` is reset to `Random.GLOBAL_RNG`,
in which case rows are only pre-shuffled if `shuffle=true` is
explicitly specified.

"""
struct CV <: ResamplingStrategy
    nfolds::Int
    shuffle::Bool
    rng::Union{Int,AbstractRNG}
    function CV(nfolds, shuffle, rng)
        nfolds > 1 || error("Must have nfolds > 1. ")
        return new(nfolds, shuffle, rng)
    end
end

# Constructor with keywords
CV(; nfolds::Int=6,  shuffle=nothing, rng=nothing) =
    CV(nfolds, shuffle_and_rng(shuffle, rng)...)

function train_test_pairs(cv::CV, rows)

    n_observations = length(rows)
    nfolds = cv.nfolds

    if cv.shuffle
        rows=shuffle!(cv.rng, collect(rows))
    end

    # number of observations per fold
    k = floor(Int, n_observations/nfolds)
    k > 0 || error("Inusufficient data for $nfolds-fold cross-validation.\n"*
                   "Try reducing nfolds. ")

    # define the (trainrows, testrows) pairs:
    firsts = 1:k:((nfolds - 1)*k + 1) # itr of first `test` rows index
    seconds = k:k:(nfolds*k)          # itr of last  `test` rows index

    ret = map(1:nfolds) do k
        f = firsts[k]
        s = seconds[k]
        k < nfolds || (s = n_observations)
        return (vcat(rows[1:(f - 1)], rows[(s + 1):end]), # trainrows
                rows[f:s])                                # testrows
    end

    return ret
end

"""
    stratified_cv = StratifiedCV(; nfolds=6,
                                   shuffle=false,
                                   rng=Random.GLOBAL_RNG)

Stratified cross-validation resampling strategy, for use in
`evaluate!`, `evaluate` and in tuning. Applies only to classification
problems (`OrderedFactor` or `Multiclass` targets).

    train_test_pairs(stratified_cv, rows, y)

Returns an `nfolds`-length iterator of `(train, test)` pairs of
vectors (row indices) where each `train` and `test` is a sub-vector of
`rows`. The `test` vectors are mutually exclusive and exhaust
`rows`. Each `train` vector is the complement of the corresponding
`test` vector.

Unlike regular cross-validation, the distribution of the levels of the
target `y` corresponding to each `train` and `test` is constrained, as
far as possible, to replicate that of `y[rows]` as a whole.

Specifically, the data is split into a number of groups on which `y`
is constant, and each individual group is resampled according to the
ordinary cross-validation strategy `CV(nfolds=nfolds)`. To obtain the
final `(train, test)` pairs of row indices, the per-group pairs are
collated in such a way that each collated `train` and `test` respects
the original order of `rows` (after shuffling, if `shuffle=true`).

Pre-shuffling of `rows` is controlled by `rng` and `shuffle`. If `rng`
is an integer, then the `StratifedCV` keyword constructor resets it to
`MersenneTwister(rng)`. Otherwise some `AbstractRNG` object is
expected.

If `rng` is left unspecified, `rng` is reset to `Random.GLOBAL_RNG`,
in which case rows are only pre-shuffled if `shuffle=true` is
explicitly specified.

"""
struct StratifiedCV <: ResamplingStrategy
    nfolds::Int
    shuffle::Bool
    rng::Union{Int,AbstractRNG}
    function StratifiedCV(nfolds, shuffle, rng)
        nfolds > 1 || error("Must have nfolds > 1. ")
        return new(nfolds, shuffle, rng)
    end
end

# Constructor with keywords
StratifiedCV(; nfolds::Int=6,  shuffle=nothing, rng=nothing) =
       StratifiedCV(nfolds, shuffle_and_rng(shuffle, rng)...)

function train_test_pairs(stratified_cv::StratifiedCV, rows, X, y)

    n_observations = length(rows)
    nfolds = stratified_cv.nfolds

    if stratified_cv.shuffle
        rows=shuffle!(stratified_cv.rng, collect(rows))
    end

    st = scitype(y)
    st <: AbstractArray{<:Finite} ||
        error("Supplied target has scitpye $st but stratified "*
              "cross-validation applies only to classification problems. ")

    freq_given_level = countmap(y[rows])
    minimum(values(freq_given_level)) >= nfolds ||
        error("The number of observations for which the target takes on a "*
              "given class must, for each class, exceed `nfolds`. Try "*
              "reducing `nfolds`. ")

    levels_seen = keys(freq_given_level) |> collect

    cv = CV(nfolds=nfolds)

    # the target is constant on each stratum, a subset of `rows`:
    class_rows = [rows[y[rows] .== c] for c in levels_seen]

    # get the cv train/test pairs for each level:
    train_test_pairs_per_level = (train_test_pairs(cv, class_rows[m])
                              for m in eachindex(levels_seen))

    # just the train rows in each level:
    trains_per_level = map(x -> first.(x),
                           train_test_pairs_per_level)

    # just the test rows in each level:
    tests_per_level  = map(x -> last.(x),
                                train_test_pairs_per_level)

    # for each fold, concatenate the train rows over levels:
    trains_per_fold = map(x->vcat(x...), zip(trains_per_level...))

    # for each fold, concatenate the test rows over levels:
    tests_per_fold = map(x->vcat(x...), zip(tests_per_level...))

    # restore ordering specified by rows:
    trains_per_fold = map(trains_per_fold) do train
        filter(in(train), rows)
    end
    tests_per_fold = map(tests_per_fold) do test
        filter(in(test), rows)
    end

    # re-assemble:
    return zip(trains_per_fold, tests_per_fold) |> collect

end

## EVALUATION TYPE

const PerformanceEvaluation = NamedTuple{(:measure, :measurement,
                               :per_fold, :per_observation)}
# pretty printing:
round3(x) = round(x, sigdigits=3)
_short(v::Vector{<:Real}) = MLJBase.short_string(v)
_short(v::Vector) = string("[", join(_short.(v), ", "), "]")
_short(::Missing) = missing

function Base.show(io::IO, ::MIME"text/plain", e::PerformanceEvaluation)
    data = hcat(e.measure, round3.(e.measurement),
                [round3.(v) for v in e.per_fold])
    header = ["_.measure", "_.measurement", "_.per_fold"]
    PrettyTables.pretty_table(io, data, header;
                              header_crayon=Crayon(bold=false),
                              alignment=:l)
    println(io, "_.per_observation = $(_short(e.per_observation))")
end

function Base.show(io::IO, e::PerformanceEvaluation)
    summary = Tuple(round3.(e.measurement))
    print(io, "PerformanceEvaluation$summary")
end

## EVALUATION METHODS

function _check_measure(model, measure, y, operation, override)

    override && (return nothing)

    T = scitype(y)

    T == Unknown && (return nothing)
    target_scitype(measure) == Unknown && (return nothing)
    prediction_type(measure) == :unknown && (return nothing)

    avoid = "\nTo override measure checks, set check_measure=false. "

    T <: target_scitype(measure) ||
        throw(ArgumentError(
            "\nscitype of target = $T but target_scitype($measure) = "*
            "$(target_scitype(measure))."*avoid))

    if model isa Probabilistic
        if operation == predict
            if prediction_type(measure) != :probabilistic
                suggestion = ""
                if target_scitype(measure) <: Finite
                    suggestion = "\nPerhaps you want to set operation="*
                    "predict_mode. "
                elseif target_scitype(measure) <: Continuous
                    suggestion = "\nPerhaps you want to set operation="*
                    "predict_mean or operation=predict_median. "
                else
                    suggestion = ""
                end
                throw(ArgumentError(
                   "\n$model <: Probabilistic but prediction_type($measure) = "*
                      ":$(prediction_type(measure)). "*suggestion*avoid))
            end
        end
    end

    model isa Deterministic && prediction_type(measure) != :deterministic &&
        throw(ArgumentError("$model <: Deterministic but "*
                            "prediction_type($measure) ="*
              ":$(prediction_type(measure))."*avoid))

    return nothing

end

function _process_weights_measures(weights, measures, mach,
                                   operation, verbosity, check_measure)

    if measures === nothing
        candidate = default_measure(mach.model)
        candidate ===  nothing && error("You need to specify measure=... ")
        _measures = [candidate, ]
    elseif !(measures isa AbstractVector)
        _measures = [measures, ]
    else
        _measures = measures
    end

    y = mach.args[2]

    [ _check_measure(mach.model, m, y, operation, !check_measure)
     for m in _measures ]

    if weights != nothing
        weights isa AbstractVector{<:Real} ||
            throw(ArgumentError("`weights` must be a `Real` vector."))
        length(weights) == nrows(y) ||
            throw(DimensionMismatch("`weights` and target "*
                                    "have different lengths. "))
        _weights = weights
    elseif  length(mach.args) == 3
        verbosity < 1 ||
            @info "Passing machine sample weights to any supported measures. "
        _weights = mach.args[3]
    else
        _weights = weights
    end

    return _weights, _measures

end

"""
    evaluate!(mach,
              resampling=CV(),
              measure=nothing,
              weights=nothing,
              operation=predict,
              n = 1,
              acceleration=default_resource(),
              force=false,
              verbosity=1)

Estimate the performance of a machine `mach` wrapping a supervised
model in data, using the specified `resampling` strategy (defaulting
to 6-fold cross-validation) and `measure`, which can be a single
measure or vector.

Do `subtypes(MLJ.ResamplingStrategy)` to obtain a list of available
resampling strategies. If `resampling` is not an object of type
`MLJ.ResamplingStrategy`, then a vector of pairs (of the form
`(train_rows, test_rows)` is expected. For example, setting

    resampling = [(1:100), (101:200)),
                   (101:200), (1:100)]

gives two-fold cross-validation using the first 200 rows of data.

The resampling strategy is applied repeatedly if `repeats > 1`. For
`resampling = CV(nfolds=5)`, for example, this generates a total of
`5n` test folds for evaluation and subsequent aggregation.

If `resampling isa MLJ.ResamplingStrategy` then one may optionally
restrict the data used in evaluation by specifying `rows`.

An optional `weights` vector may be passed for measures that support
sample weights (`MLJ.supports_weights(measure) == true`), which is
ignored by those that don't.

*Important:* If `mach` already wraps sample weights `w` (as in `mach =
machine(model, X, y, w)`) then these weights, which are used for
*training*, are automatically passed to the measures for
evaluation. However, for evaluation purposes, any `weights` specified
as a keyword argument will take precedence over `w`.

User-defined measures are supported; see the manual for details.

If no measure is specified, then `default_measure(mach.model)` is
used, unless this default is `nothing` and an error is thrown.

The `acceleration` keyword argument is used to specify the compute resource (a
subtype of `ComputationalResources.AbstractResource`) that will be used to
accelerate/parallelize the resampling operation.

Although evaluate! is mutating, `mach.model` and `mach.args` are
untouched.

### Return value

A property-accessible object of type `PerformanceEvaluation` with
these properties:

- `measure`: the vector of specified measures

- `measurements`: the corresponding measurements, aggregated across the
  test folds using the aggregation method defined for each measure (do
  `aggregation(measure)` to inspect)

- `per_fold`: a vector of vectors of individual test fold evaluations
  (one vector per measure)

- `per_observation`: a vector of vectors of individual observation
  evaluations of those measures for which
  `reports_each_observation(measure)` is true, which is otherwise
  reported `missing`.

See also [`evaluate`](@ref)

"""
function evaluate!(mach::Machine{<:Supervised};
                   resampling=CV(),
                   measures=nothing, measure=measures, weights=nothing,
                   operation=predict, acceleration=default_resource(),
                   rows=nothing, repeats=1, force=false,
                   check_measure=true, verbosity=1)

    # this method just checks validity of options, preprocess the
    # weights and measures, and dispatches a strategy-specific
    # `evaluate!`

    repeats > 0 || error("Need n > 0. ")

    if resampling isa TrainTestPairs
        if rows !== nothing
            error("You cannot specify `rows` unless `resampling "*
                  "isa MLJ.ResamplingStrategy` is true. ")
        end
        if repeats != 1 && verbosity > 0
            @warn "repeats > 1 not supported unless "*
            "`resampling<:ResamplingStrategy. "
        end
    end

    _weights, _measures =
        _process_weights_measures(weights, measure, mach,
                                  operation, verbosity, check_measure)

    if verbosity >= 0 && weights !== nothing
        unsupported = filter(_measures) do m
            !supports_weights(m)
        end
        if !isempty(unsupported)
            unsupported_as_string = string(unsupported[1])
            unsupported_as_string *=
                reduce(*, [string(", ", m) for m in unsupported[2:end]])
                @warn "Sample weights ignored in evaluations of the following"*
            " measures, as unsupported: \n$unsupported_as_string "
        end
    end

    evaluate!(mach, resampling, _weights, rows, verbosity, repeats,
                   _measures, operation, acceleration, force)

end

"""
    evaluate(model, X, y; measure=nothing, options...)
    evaluate(model, X, y, w; measure=nothing, options...)

Evaluate the performance of a supervised model `model` on input data
`X` and target `y`, optionally specifying sample weights `w` for
training, where supported. The same weights are passed to measures
that support sample weights, unless this behaviour is overridden by
explicitly specifying the option `weights=...`.

See the machine version `evaluate!` for the complete list of options.

"""
evaluate(model::Supervised, args...; kwargs...) =
    evaluate!(machine(model, args...); kwargs...)

const AbstractRow = Union{AbstractVector{<:Integer}, Colon}
const TrainTestPair = Tuple{AbstractRow,AbstractRow}
const TrainTestPairs = AbstractVector{<:TrainTestPair}

function _evaluate!(func::Function, res::CPU1, nfolds, verbosity)
    p = Progress(nfolds + 1, dt=0, desc="Evaluating over $nfolds folds: ",
                 barglyphs=BarGlyphs("[=> ]"), barlen=25, color=:yellow)
    verbosity > 0 && next!(p)
    return reduce(vcat, (func(k, p, verbosity) for k in 1:nfolds))
end
function _evaluate!(func::Function, res::CPUProcesses, nfolds, verbosity)
    # TODO: use pmap here ?:
    return @distributed vcat for k in 1:nfolds
        func(k)
    end
end
@static if VERSION >= v"1.3.0-DEV.573"
    function _evaluate!(func::Function, res::CPUThreads, nfolds, verbosity)
        task_vec = [Threads.@spawn func(k) for k in 1:nfolds]
        return reduce(vcat, fetch.(task_vec))
    end
end

# Evaluation when resampling is a TrainTestPairs (core evaluator):
function evaluate!(mach::Machine, resampling, weights,
                   rows, verbosity, repeats,
                   measures, operation, acceleration, force)

    # Note: `rows` and `n` are ignored here

    resampling isa TrainTestPairs ||
        error("`resampling` must be an "*
              "`MLJ.ResamplingStrategy` or tuple of pairs "*
              "of the form `(train_rows, test_rows)`")

    X = mach.args[1]
    y = mach.args[2]

    nfolds = length(resampling)

    nmeasures = length(measures)

    function get_measurements(k)
        train, test = resampling[k]
        fit!(mach; rows=train, verbosity=verbosity-1, force=force)
        Xtest = selectrows(X, test)
        ytest = selectrows(y, test)
        if weights == nothing
            wtest = nothing
        else
            wtest = weights[test]
        end
        yhat = operation(mach, Xtest)
        return [value(m, yhat, Xtest, ytest, wtest)
                for m in measures]
    end
    function get_measurements(k, p, verbosity) # p = progress meter
        ret = get_measurements(k)
        verbosity > 0 && next!(p)
        return ret
    end

    measurements_flat = if acceleration isa CPUProcesses
        ## TODO: progress meter for distributed case
        if verbosity > 0
            @info "Distributing cross-validation computation " *
                  "among $(nworkers()) workers."
        end
    end
    measurements_flat =
        _evaluate!(get_measurements, acceleration, nfolds, verbosity)

    # in the following rows=folds, columns=measures:
    measurements_matrix = permutedims(
        reshape(measurements_flat, (nmeasures, nfolds)))

    # measurements for each observation:
    per_observation = map(1:nmeasures) do k
        m = measures[k]
        if reports_each_observation(m)
            [measurements_matrix[:,k]...]
        else
            missing
        end
    end

    # measurements for each fold:
    per_fold = map(1:nmeasures) do k
        m = measures[k]
        if reports_each_observation(m)
            broadcast(MLJBase.aggregate, per_observation[k], [m,])
        else
            [measurements_matrix[:,k]...]
        end
    end

    # overall aggregates:
    per_measure = map(1:nmeasures) do k
        m = measures[k]
        MLJBase.aggregate(per_fold[k], m)
    end

    ret = (measure=measures,
           measurement=per_measure,
           per_fold=per_fold,
           per_observation=per_observation)

    return ret

end

function actual_rows(rows, N, verbosity)
    unspecified_rows = (rows === nothing)
    _rows = unspecified_rows ? (1:N) : rows
    unspecified_rows || @info "Creating subsamples from a subset of all rows. "
    return _rows
end

# Evaluation when resampling is a ResamplingStrategy:
function evaluate!(mach::Machine, resampling::ResamplingStrategy,
                   weights, rows, verbosity, repeats, args...)

    y = mach.args[2]
    _rows = actual_rows(rows, length(y), verbosity)

    repeated_train_test_pairs =
        vcat([train_test_pairs(resampling, _rows, mach.args...)
              for i in 1:repeats]...)

    return evaluate!(mach::Machine,
                     repeated_train_test_pairs,
                     weights, nothing, verbosity, repeats, args...)

end

## RESAMPLER - A MODEL WRAPPER WITH `evaluate` OPERATION

"""
    resampler = Resampler(model=ConstantRegressor(),
                          resampling=CV(),
                          measure=nothing,
                          weights=nothing,
                          operation=predict,
                          repeats = 1,
                          acceleration=default_resource(),
                          check_measure=true)

Resampling model wrapper, used internally by the `fit` method of
`TunedModel` instances. See [`evaluate!](@ref) for options. Not
intended for general use.

Given a machine `mach = machine(resampler, args...)` one obtains a
performance evaluation of the specified `model`, performed according
to the prescribed `resampling` strategy and other parameters, using
data `args...`, by calling `fit!(mach)` followed by
`evaluate(mach)`. The advantage over using `evaluate(model, X, y)` is
that the latter call always calls `fit` on the `model` but
`fit!(mach)` only calls `update` after the first call.

The sample `weights` are passed to the specified performance
measures that support weights for evaluation.

*Important:* If `weights` are left unspecified, then any weight vector
`w` used in constructing the resampler machine, as in
`resampler_machine = machine(resampler, X, y, w)` (which is then used
in *training* the model) will also be used in evaluation.

"""
mutable struct Resampler{S,M<:Supervised} <: Supervised
    model::M
    resampling::S # resampling strategy
    measure
    weights::Union{Nothing,AbstractVector{<:Real}}
    operation
    acceleration::AbstractResource
    check_measure::Bool
    repeats::Int
end

MLJBase.is_wrapper(::Type{<:Resampler}) = true
MLJBase.supports_weights(::Type{<:Resampler{<:Any,M}}) where M =
    supports_weights(M)
MLJBase.is_pure_julia(::Type{<:Resampler}) = true

function MLJBase.clean!(resampler::Resampler)
    warning = ""
    if resampler.measure === nothing
        measure = default_measure(resampler.model)
        if measure === nothing
            error("No default measure known for $(resampler.model). "*
                  "You must specify measure=... ")
        else
            warning *= "No `measure` specified. "*
            "Setting `measure=$measure`. "
        end
    end
    return warning
end

function Resampler(; model=ConstantRegressor(), resampling=CV(),
            measure=nothing, weights=nothing, operation=predict,
            acceleration=default_resource(), check_measure=true, repeats=1)

    resampler = Resampler(model, resampling, measure, weights, operation,
                          acceleration, check_measure, repeats)
    message = MLJBase.clean!(resampler)
    isempty(message) || @warn message

    return resampler

end

function MLJBase.fit(resampler::Resampler, verbosity::Int, args...)

    mach = machine(resampler.model, args...)

    weights, measures =
        _process_weights_measures(resampler.weights, resampler.measure,
                                  mach, resampler.operation,
                                  verbosity, resampler.check_measure)

    fitresult = evaluate!(mach, resampler.resampling,
                          weights, nothing, verbosity - 1, resampler.repeats,
                          measures, resampler.operation,
                          resampler.acceleration, false)

    cache = (mach, deepcopy(resampler.resampling))
    report = NamedTuple()

    return fitresult, cache, report

end

# in special case of holdout, we can reuse the underlying model's
# machine, provided the training_fraction has not changed:
function MLJBase.update(resampler::Resampler{Holdout},
                        verbosity::Int, fitresult, cache, args...)

    old_mach, old_resampling = cache

    if old_resampling.fraction_train == resampler.resampling.fraction_train
        mach = old_mach
    else
        mach = machine(resampler.model, args...)
        cache = (mach, deepcopy(resampler.resampling))
    end

    weights, measures =
        _process_weights_measures(resampler.weights, resampler.measure,
                                  mach, resampler.operation,
                                  verbosity, resampler.check_measure)
    mach.model = resampler.model
    fitresult = evaluate!(mach, resampler.resampling,
                          weights, nothing, verbosity - 1, resampler.repeats,
                          measures, resampler.operation,
                          resampler.acceleration, false)

    report = NamedTuple

    return fitresult, cache, report

end

MLJBase.input_scitype(::Type{<:Resampler{S,M}}) where {S,M} =
    MLJBase.input_scitype(M)
MLJBase.target_scitype(::Type{<:Resampler{S,M}}) where {S,M} =
    MLJBase.target_scitype(M)
MLJBase.package_name(::Type{<:Resampler}) = "MLJBase"

MLJBase.load_path(::Type{<:Resampler}) = "MLJBase.Resampler"

evaluate(resampler::Resampler, fitresult) = fitresult

function evaluate(machine::AbstractMachine{<:Resampler})
    if isdefined(machine, :fitresult)
        return evaluate(machine.model, machine.fitresult)
    else
        throw(error("$machine has not been trained."))
    end
end


#=

    Reading input

=#
function read_input()

    pass=[]
    policy=[]
    letter=[]
    open("./input", "r") do f
        while ! eof(f)
            s::String = readline(f)
            entry = split(s," ")
            push!(pass,entry[3])
            push!(letter,entry[2][1])

            policy_split = split(entry[1],"-")
            policy_tuple = (parse(Int32,policy_split[1]),
                            parse(Int32,policy_split[2]))
            push!(policy,policy_tuple)
        end
    end

    return pass, policy, letter
end

input = read_input()
pass = input[1]
policy = input[2]
letter = input[3]

#=

--- Day 2: Password Philosophy ---

Your flight departs in a few days from the coastal airport; the easiest way down to
the coast from here is via toboggan.

The shopkeeper at the North Pole Toboggan Rental Shop is having a bad day. "Something's
wrong with our computers; we can't log in!" You ask if you can take a look.

Their password database seems to be a little corrupted: some of the passwords wouldn't
have been allowed by the Official Toboggan Corporate Policy that was in effect when they
were chosen.

To try to debug the problem, they have created a list (your puzzle input) of passwords
(according to the corrupted database) and the corporate policy when that password was set.

For example, suppose you have the following list:

1-3 a: abcde
1-3 b: cdefg
2-9 c: ccccccccc

Each line gives the password policy and then the password. The password policy indicates
the lowest and highest number of times a given letter must appear for the password to be
valid. For example, 1-3 a means that the password must contain a at least 1 time and at
most 3 times.

In the above example, 2 passwords are valid. The middle password, cdefg, is not; it contains
no instances of b, but needs at least 1. The first and third passwords are valid: they contain
one a or nine c, both within the limits of their respective policies.

How many passwords are valid according to their policies?

=#

function is_valid(policy, letter, password)
    letter_count = count(==(letter), password)
    if letter_count<policy[1]
        return false
    elseif letter_count>policy[2]
        return false
    else
        return true
    end
end

function count_valid_passwords(policy, letter, pass)
    count_valid = 0
    for i in 1:size(pass,1)
        if is_valid(policy[i], letter[i], pass[i])
            count_valid+=1
        end
    end
    return count_valid
end

println(size(policy,1),", ",size(letter,1), ", ", size(pass,1))
println("Valid passwords found: ", count_valid_passwords(policy, letter, pass))

#=
--- Part Two ---

While it appears you validated the passwords correctly, they don't seem to be
what the Official Toboggan Corporate Authentication System is expecting.

The shopkeeper suddenly realizes that he just accidentally explained the password
policy rules from his old job at the sled rental place down the street! The Official
Toboggan Corporate Policy actually works a little differently.

Each policy actually describes two positions in the password, where 1 means the
first character, 2 means the second character, and so on. (Be careful; Toboggan
Corporate Policies have no concept of "index zero"!) Exactly one of these positions
must contain the given letter. Other occurrences of the letter are irrelevant for
the purposes of policy enforcement.

Given the same example list from above:

    1-3 a: abcde is valid: position 1 contains a and position 3 does not.
    1-3 b: cdefg is invalid: neither position 1 nor position 3 contains b.
    2-9 c: ccccccccc is invalid: both position 2 and position 9 contain c.

How many passwords are valid according to the new interpretation of the policies?
=#

function is_valid(policy, letter, password)
    in_fst_pos = (password[policy[1]] == letter)
    in_snd_pos = (password[policy[2]] == letter)

    if in_fst_pos
        if in_snd_pos
            return false
        else
            return true
        end
    elseif in_snd_pos
        return true
    else
        return false
    end
end

function count_valid_passwords(policy, letter, pass)
    count_valid = 0
    for i in 1:size(pass,1)
        if is_valid(policy[i], letter[i], pass[i])
            count_valid+=1
        end
    end
    return count_valid
end

println("Valid passwords found with second policy: ", count_valid_passwords(policy, letter, pass))


### A Pluto.jl notebook ###
# v0.18.2

using Markdown
using InteractiveUtils

# ╔═╡ c3530026-5a87-4447-88f5-fc744cb48ff2
begin
	using PlutoUI
end

# ╔═╡ 2c895e14-e71a-435d-86d4-c1b101877b3c
md"""
### The End of the Beginning: Whole-Cell Metabolic Engineering

In the first lecture, we introduced metabolic engineering. Today, we'll close the loop at put together all we learned. Metabolic engineering is the practice of optimizing genetic and regulatory processes within cells to increase the cell's production of a desired small molecule or protein product of interest.

Metabolic engineers manipulate the biochemical networks cells use to convert raw materials into molecules necessary for the cell's survival. Metabolic engineering specifically seeks to:

1. Mathematically model biochemical networks, calculate the yield (product divided by substrate) of valuable products and identify parts of the network that constrain the production of these products of interest.
1. Use genetic engineering techniques to modify the biochemical network to relieve constraints limiting production. Metabolic engineers can then model the modified network to calculate the new product yield and identify new constraints (back to 1).

In this (the final lecture of Part I of the course), we'll:

* Incorporate the costs of transcription and translation into the flux balance analysis problem
* Introduce a workaround (often used in practice) to account for the metabolic cost of gene expression
* Close the loop: using a metabolic model and flux balance to optimize glycan production in _E.coli_.

"""

# ╔═╡ ce5c8e9c-7652-4c3e-98a7-98e3567dc695
md"""
### Integrating the Cost and Logic of Gene Expression With Flux Balance Analysis

The [Allen and Palsson study](https://pubmed.ncbi.nlm.nih.gov/12453446/) gave us a roadmap of integrating the cost of gene expression with the metabolic operation of the cell. Further, [Vilkhovoy et al.](https://pubmed.ncbi.nlm.nih.gov/29944340/) showed how we could do this in a cell-free system. However, how do you account for gene expression in metabolic models at the whole-genome scale?

* [Thiele I, Jamshidi N, Fleming RM, Palsson BØ. Genome-scale reconstruction of Escherichia coli's transcriptional and translational machinery: a knowledge base, its mathematical formulation, and its functional characterization. PLoS Comput Biol. 2009;5(3):e1000312. doi:10.1371/journal.pcbi.1000312](https://www.ncbi.nlm.nih.gov/labs/pmc/articles/PMC2648898/?report=classic)
* [Lerman JA, Hyduke DR, Latif H, Portnoy VA, Lewis NE, Orth JD, Schrimpe-Rutledge AC, Smith RD, Adkins JN, Zengler K, Palsson BO. In silico method for modeling metabolism and gene product expression at genome scale. Nat Commun. 2012 Jul 3;3:929. doi: 10.1038/ncomms1928. PMID: 22760628; PMCID: PMC3827721.](https://pubmed.ncbi.nlm.nih.gov/22760628/)
"""

# ╔═╡ bfd134fe-b311-4768-a53b-2c5c4c1b121a
md"""
##### Aside: A question of timescales
One open question is whether we need to solve the gene expression and flux estimation problem simultaneously. The time scale of enzyme activity (and hence metabolic flux) is much faster than gene expression. Why?

* The $k_{cat}$ for [phosphofructokinase-2 (pfk) in _E.coli_ is 9240 s$^{-1}$](https://bionumbers.hms.harvard.edu/bionumber.aspx?id=104955&ver=7&trm=kcat+pfk+in+E.+coli&org=). Thus, pfk has a characteristic time scale of $\tau\sim{k_{cat}}^{-1}$ or about 1$\times$10$^{-4}$ s.
* The $k_{cat}$ for RNAP polymerase is $k_{cat}$=$e_{x}L^{-1}$. If we assume [$e_{x}\sim{35}$ nt/s](https://bionumbers.hms.harvard.edu/bionumber.aspx?id=111871&ver=1&trm=elongation+rate+RNAP+E.coli+&org=) and average read length [$L\sim{924}$ nt](https://bionumbers.hms.harvard.edu/bionumber.aspx?id=111922&ver=3&trm=avarge+gene+length+in+E.+coli&org=), this gives $k_{cat}\sim$ 0.038 s$^{-1}$ or a timescale of $\tau\sim$26.4 s.

Thus, the time scale of metabolic flux (e.g., the timescale of enzyme activity) is approximately 5-orders of magnitude faster than gene expression. Therefore, gene expression seems constant  from the perspective of metabolic flux; hence, we treat these scales separately in traditional models.
"""

# ╔═╡ 8041cb6c-b80f-4e81-8d1d-5a2e9942d809
md"""
##### Aside: How do you account for macromolecular synthesis without the E/ME formulation?
The E/ME formulation of Palsson and coworkers is detailed and requires more time investment than perhaps you're willing to give in a metabolic engineering application. How do you account for the energy and resource cost of producing enzymes (and all the other machinery) required to make a cell?

The traditional way metabolic engineers address this question is to treat cell mass formation as just another reaction, e.g., the growth reaction $\mu$:

$$\left\{precursors\right\}~\rightarrow~cell$$

where each cellmass precursor has some stoichiometric coefficient $\sigma_{i,\mu}$ which describes how much of precursor $i$ is consumed (or produced) during cellmass formation. This formulation leads to species material balances of the form (assuming cellmass specific units, in a constant volume batch culture):

$$\frac{dx_{i}}{dt} = \sum_{j=1}^{\mathcal{R}}\sigma_{ij}\hat{r}_{j} - \left(x_{i}-\sigma_{i,\mu}\right)\mu\qquad{i=1,2,\dots,\mathcal{M}}$$

At steady state, for example in a flux balance analysis problem, these balances form the species constraints:

$$\sum_{j=1}^{\mathcal{R}}\sigma_{ij}\hat{r}_{j} - \left(x_{i}-\sigma_{i,\mu}\right)\mu = 0\qquad{i=1,2,\dots,\mathcal{M}}$$

Palsson and coworkers gave an example cell mass reaction for _E.coli_ in:

* [Orth JD, Fleming RM, Palsson BØ. Reconstruction and Use of Microbial Metabolic Networks: the Core Escherichia coli Metabolic Model as an Educational Guide. EcoSal Plus. 2010 Sep;4(1). doi: 10.1128/ecosalplus.10.2.1. PMID: 26443778.](https://pubmed.ncbi.nlm.nih.gov/26443778/)

A recent paper on sampling uncertain biomass stoichiometric coefficients:

* [Dinh et al. (2022) Quantifying the propagation of parametric uncertainty on flux balance analysis. Metabolic Engineering, 69:26-39; https://doi.org/10.1016/j.ymben.2021.10.012](https://www.sciencedirect.com/science/article/pii/S1096717621001634)

"""

# ╔═╡ 06c3eafa-10b3-4a47-9a06-39f6abd29db3
md"""
### Using flux balance analysis to improve metabolic performance
Ultimately a metabolic engineer uses a model to improve the performance of a production host or cell-free system. There is a huge variety of design approaches to improve metabolic function; let's look at a recent one from Cornell:

* [Wayman JA, Glasscock C, Mansell TJ, DeLisa MP, Varner JD. Improving designer glycan production in Escherichia coli through model-guided metabolic engineering. Metab Eng Commun. 2019;9:e00088. Published 2019 Mar 29. doi:10.1016/j.mec.2019.e00088](https://www.ncbi.nlm.nih.gov/labs/pmc/articles/PMC6454127/)

"""

# ╔═╡ b15b540b-8f13-4f21-b2da-510d34143b33
md"""
### Summary and conclusions

Today, we:

* Incorporated the costs of transcription and translation into the flux balance analysis problem
* Introduced a workaround (often used in practice) to account for the metabolic cost of gene expression
* Closed the loop: using a metabolic model and flux balance to optimize glycan production in _E.coli_.
"""

# ╔═╡ a818128b-d642-49e0-8c7c-db36b37ea882
md"""
### Alas, if we just had some more time, it would have been cool to talk about:

##### So many interesting tools:

* [Lewis NE, Nagarajan H, Palsson BO. Constraining the metabolic genotype-phenotype relationship using a phylogeny of in silico methods. Nat Rev Microbiol. 2012 Feb 27;10(4):291-305. doi: 10.1038/nrmicro2737. PMID: 22367118; PMCID: PMC3536058.](https://pubmed.ncbi.nlm.nih.gov/22367118/)

##### Better constraints give better flux estimates:

* [Buescher JM, Antoniewicz MR, Boros LG, et al. A roadmap for interpreting (13)C metabolite labeling patterns from cells. Curr Opin Biotechnol. 2015;34:189-201. doi:10.1016/j.copbio.2015.02.003](https://www.ncbi.nlm.nih.gov/labs/pmc/articles/PMC4552607/)

##### Flux balance analysis prediction of mutant response is not correct:

* [Segrè D, Vitkup D, Church GM. Analysis of optimality in natural and perturbed metabolic networks. Proc Natl Acad Sci U S A. 2002 Nov 12;99(23):15112-7. doi: 10.1073/pnas.232349399. Epub 2002 Nov 1. PMID: 12415116; PMCID: PMC137552.](https://pubmed.ncbi.nlm.nih.gov/12415116/)

##### Molecular crowding plays a role in metabolism (non-representative random sample):

* [Beg QK, Vazquez A, Ernst J, de Menezes MA, Bar-Joseph Z, Barabási AL, Oltvai ZN. Intracellular crowding defines the mode and sequence of substrate uptake by Escherichia coli and constrains its metabolic activity. Proc Natl Acad Sci U S A. 2007 Jul 31;104(31):12663-8. doi: 10.1073/pnas.0609845104. Epub 2007 Jul 24. PMID: 17652176; PMCID: PMC1937523.](https://pubmed.ncbi.nlm.nih.gov/17652176/)'
* [Conrado RJ, Mansell TJ, Varner JD, DeLisa MP. Stochastic reaction-diffusion simulation of enzyme compartmentalization reveals improved catalytic efficiency for a synthetic metabolic pathway. Metab Eng. 2007 Jul;9(4):355-63. doi: 10.1016/j.ymben.2007.05.002. Epub 2007 May 26. PMID: 17601761.](https://pubmed.ncbi.nlm.nih.gov/17601761/)
* [Klumpp S, Scott M, Pedersen S, Hwa T. Molecular crowding limits translation and cell growth. Proc Natl Acad Sci U S A. 2013 Oct 15;110(42):16754-9. doi: 10.1073/pnas.1310377110. Epub 2013 Sep 30. PMID: 24082144; PMCID: PMC3801028.](https://pubmed.ncbi.nlm.nih.gov/22760628/)

"""

# ╔═╡ 84d8a1c4-483c-4cc2-bdd0-7c88d26fd319
TableOfContents(title="📚 Lecture Outline", indent=true, depth=5, aside=true)

# ╔═╡ 7988263a-a5f5-11ec-3836-150105d04264
html"""
<style>
main {
    max-width: 900px;
    width: 70%;
    margin: auto;
    font-family: "Roboto, monospace";
}

a {
    color: blue;
    text-decoration: none;
}
</style>"""

# ╔═╡ dee24b55-a5c5-4a47-9e1c-88ea0cf1cd61
html"""
<script>
	// initialize -
	var section = 0;
	var subsection = 0;
	var subsubsection = 0;
	var headers = document.querySelectorAll('h3, h5, h6');

	// main loop -
	for (var i=0; i < headers.length; i++) {

		var header = headers[i];
	    var text = header.innerText;
	    var original = header.getAttribute("text-original");
	    if (original === null) {

			// Save original header text
	        header.setAttribute("text-original", text);
	    } else {

			// Replace with original text before adding section number
	        text = header.getAttribute("text-original");
	    }

	    var numbering = "";
	    switch (header.tagName) {
	        case 'H3':
	            section += 1;
	            numbering = section + ".";
	            subsection = 0;
	            break;
	        case 'H5':
	            subsection += 1;
	            numbering = section + "." + subsection;
	            break;
			case 'H6':
	            subsubsection += 1;
	            numbering = section + "." + subsection + "." + subsubsection;
	            break;
	    }
		// update the header text
		header.innerText = numbering + " " + text;
	};
</script>"""

# ╔═╡ 00000000-0000-0000-0000-000000000001
PLUTO_PROJECT_TOML_CONTENTS = """
[deps]
PlutoUI = "7f904dfe-b85e-4ff6-b463-dae2292396a8"

[compat]
PlutoUI = "~0.7.37"
"""

# ╔═╡ 00000000-0000-0000-0000-000000000002
PLUTO_MANIFEST_TOML_CONTENTS = """
# This file is machine-generated - editing it directly is not advised

julia_version = "1.7.2"
manifest_format = "2.0"

[[deps.AbstractPlutoDingetjes]]
deps = ["Pkg"]
git-tree-sha1 = "8eaf9f1b4921132a4cff3f36a1d9ba923b14a481"
uuid = "6e696c72-6542-2067-7265-42206c756150"
version = "1.1.4"

[[deps.ArgTools]]
uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"

[[deps.Artifacts]]
uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"

[[deps.Base64]]
uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"

[[deps.ColorTypes]]
deps = ["FixedPointNumbers", "Random"]
git-tree-sha1 = "024fe24d83e4a5bf5fc80501a314ce0d1aa35597"
uuid = "3da002f7-5984-5a60-b8a6-cbb66c0b333f"
version = "0.11.0"

[[deps.CompilerSupportLibraries_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae"

[[deps.Dates]]
deps = ["Printf"]
uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"

[[deps.Downloads]]
deps = ["ArgTools", "LibCURL", "NetworkOptions"]
uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"

[[deps.FixedPointNumbers]]
deps = ["Statistics"]
git-tree-sha1 = "335bfdceacc84c5cdf16aadc768aa5ddfc5383cc"
uuid = "53c48c17-4a7d-5ca2-90c5-79b7896eea93"
version = "0.8.4"

[[deps.Hyperscript]]
deps = ["Test"]
git-tree-sha1 = "8d511d5b81240fc8e6802386302675bdf47737b9"
uuid = "47d2ed2b-36de-50cf-bf87-49c2cf4b8b91"
version = "0.0.4"

[[deps.HypertextLiteral]]
git-tree-sha1 = "2b078b5a615c6c0396c77810d92ee8c6f470d238"
uuid = "ac1192a8-f4b3-4bfe-ba22-af5b92cd3ab2"
version = "0.9.3"

[[deps.IOCapture]]
deps = ["Logging", "Random"]
git-tree-sha1 = "f7be53659ab06ddc986428d3a9dcc95f6fa6705a"
uuid = "b5f81e59-6552-4d32-b1f0-c071b021bf89"
version = "0.2.2"

[[deps.InteractiveUtils]]
deps = ["Markdown"]
uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"

[[deps.JSON]]
deps = ["Dates", "Mmap", "Parsers", "Unicode"]
git-tree-sha1 = "3c837543ddb02250ef42f4738347454f95079d4e"
uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
version = "0.21.3"

[[deps.LibCURL]]
deps = ["LibCURL_jll", "MozillaCACerts_jll"]
uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"

[[deps.LibCURL_jll]]
deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"]
uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0"

[[deps.LibGit2]]
deps = ["Base64", "NetworkOptions", "Printf", "SHA"]
uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"

[[deps.LibSSH2_jll]]
deps = ["Artifacts", "Libdl", "MbedTLS_jll"]
uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8"

[[deps.Libdl]]
uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"

[[deps.LinearAlgebra]]
deps = ["Libdl", "libblastrampoline_jll"]
uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"

[[deps.Logging]]
uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"

[[deps.Markdown]]
deps = ["Base64"]
uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"

[[deps.MbedTLS_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"

[[deps.Mmap]]
uuid = "a63ad114-7e13-5084-954f-fe012c677804"

[[deps.MozillaCACerts_jll]]
uuid = "14a3606d-f60d-562e-9121-12d972cd8159"

[[deps.NetworkOptions]]
uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"

[[deps.OpenBLAS_jll]]
deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"]
uuid = "4536629a-c528-5b80-bd46-f80d51c5b363"

[[deps.Parsers]]
deps = ["Dates"]
git-tree-sha1 = "85b5da0fa43588c75bb1ff986493443f821c70b7"
uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
version = "2.2.3"

[[deps.Pkg]]
deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"]
uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"

[[deps.PlutoUI]]
deps = ["AbstractPlutoDingetjes", "Base64", "ColorTypes", "Dates", "Hyperscript", "HypertextLiteral", "IOCapture", "InteractiveUtils", "JSON", "Logging", "Markdown", "Random", "Reexport", "UUIDs"]
git-tree-sha1 = "bf0a1121af131d9974241ba53f601211e9303a9e"
uuid = "7f904dfe-b85e-4ff6-b463-dae2292396a8"
version = "0.7.37"

[[deps.Printf]]
deps = ["Unicode"]
uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"

[[deps.REPL]]
deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"]
uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"

[[deps.Random]]
deps = ["SHA", "Serialization"]
uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"

[[deps.Reexport]]
git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b"
uuid = "189a3867-3050-52da-a836-e630ba90ab69"
version = "1.2.2"

[[deps.SHA]]
uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"

[[deps.Serialization]]
uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"

[[deps.Sockets]]
uuid = "6462fe0b-24de-5631-8697-dd941f90decc"

[[deps.SparseArrays]]
deps = ["LinearAlgebra", "Random"]
uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"

[[deps.Statistics]]
deps = ["LinearAlgebra", "SparseArrays"]
uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"

[[deps.TOML]]
deps = ["Dates"]
uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"

[[deps.Tar]]
deps = ["ArgTools", "SHA"]
uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"

[[deps.Test]]
deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

[[deps.UUIDs]]
deps = ["Random", "SHA"]
uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"

[[deps.Unicode]]
uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"

[[deps.Zlib_jll]]
deps = ["Libdl"]
uuid = "83775a58-1f1d-513f-b197-d71354ab007a"

[[deps.libblastrampoline_jll]]
deps = ["Artifacts", "Libdl", "OpenBLAS_jll"]
uuid = "8e850b90-86db-534c-a0d3-1478176c7d93"

[[deps.nghttp2_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d"

[[deps.p7zip_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
"""

# ╔═╡ Cell order:
# ╟─2c895e14-e71a-435d-86d4-c1b101877b3c
# ╟─ce5c8e9c-7652-4c3e-98a7-98e3567dc695
# ╟─bfd134fe-b311-4768-a53b-2c5c4c1b121a
# ╟─8041cb6c-b80f-4e81-8d1d-5a2e9942d809
# ╟─06c3eafa-10b3-4a47-9a06-39f6abd29db3
# ╟─b15b540b-8f13-4f21-b2da-510d34143b33
# ╟─a818128b-d642-49e0-8c7c-db36b37ea882
# ╠═84d8a1c4-483c-4cc2-bdd0-7c88d26fd319
# ╠═c3530026-5a87-4447-88f5-fc744cb48ff2
# ╟─7988263a-a5f5-11ec-3836-150105d04264
# ╟─dee24b55-a5c5-4a47-9e1c-88ea0cf1cd61
# ╟─00000000-0000-0000-0000-000000000001
# ╟─00000000-0000-0000-0000-000000000002


export numstring

"""
`numstring(x, n=2)`

Left pads a number (`x`) with zeros to a total of `n` charaters.

## Example
```julia-repl
julia> numstring(11, 2)
"11"

julia> numstring(11, 3)
"011"

julia> numstring(11, 4)
"0011"
```
"""
numstring(x, n=2) = string(10^n + x)[2:end]


struct KnotVector{T,P<:AbstractVector{T}} <: AbstractVector{T}
    parent::P
    front::Int
    back::Int

    @inline function KnotVector{T,P}(parent, front, back) where {T,P<:AbstractVector{T}}
        @boundscheck begin
            front ≥ 0 || throw(DomainError(front, "front padding must be non-negative."))
            back  ≥ 0 || throw(DomainError(back,  "back padding must be non-negative."))
            has_offset_axes(parent) && throw(ArgumentError("parent array must not have offset axes."))
            isempty(parent) && throw(ArgumentError("parent array must not be empty."))
        end
        new(parent, front, back)
    end
end

"""
    KnotVector(parent::AbstractVector, front[, back=front])

Create a view into `parent` where the first element of `parent` is appended `front` times at
the front and the last element `back` times at the back. (not exported)

# Examples

```jldoctest
julia> BSplines.KnotVector(0:4, 2)
9-element $(BSplines.KnotVector{Int64, UnitRange{Int64}}):
 0
 0
 0
 1
 2
 3
 4
 4
 4

julia> BSplines.KnotVector(["first", "second", "last"], 2, 3)
8-element $(BSplines.KnotVector{String, Vector{String}}):
 "first"
 "first"
 "first"
 "second"
 "last"
 "last"
 "last"
 "last"
```
"""
@propagate_inbounds KnotVector(parent::AbstractVector, front, back=front) =
    KnotVector{eltype(parent), typeof(parent)}(parent, front, back)
@propagate_inbounds KnotVector(p::KnotVector, front, back=front) =
    KnotVector(parent(p), front + p.front, back + p.back)

@inline @propagate_inbounds function Base.getindex(p::KnotVector, i::Int)
    @boundscheck checkbounds(p, i)
    @inbounds parent(p)[clamp(i-p.front, 1, length(parent(p)))]
end
Base.getindex(p::KnotVector, ::Colon) = copy(p)

Base.IndexStyle(p::KnotVector) = IndexLinear()
Base.parent(p::KnotVector) = p.parent
Base.size(p::KnotVector) = (length(parent(p)) + p.front + p.back,)

Base.copy(p::KnotVector) = @inbounds KnotVector(copy(parent(p)), p.front, p.back)

Base.first(p::KnotVector) = @inbounds first(parent(p))
Base.last(p::KnotVector) = @inbounds last(parent(p))
Base.issorted(p::KnotVector) = issorted(parent(p))

Base.allunique(p::KnotVector) = iszero(p.front) && iszero(p.back) && allunique(parent(p))
Base.unique(p::KnotVector) = unique(parent(p))

function intervalindex(p::KnotVector, x)
    index = intervalindex(parent(p), x)
    index === nothing && return nothing
    return index + p.front
end

function intervalindex(p::KnotVector, x, start::Integer)
    checkbounds(p, start)
    index = intervalindex(parent(p), x, clamp(start-p.front, 1, length(parent(p))))
    index === nothing && return nothing
    return index + p.front
end

struct StandardBasisVector{T<:Number} <: AbstractVector{T}
    length::Int
    index::Int

    @inline function StandardBasisVector{T}(length, index) where T
        @boundscheck if !(1 ≤ index ≤ length)
            throw(DomainError(index, "Index not in range 1:$length."))
        end
        new(length, index)
    end
end

"""
    StandardBasisVector([T=Bool,] length, i)

Create a vector of the specified `length` where the `i`-th element is equal to `1` and all
other elements are equal to `0`. The elements are of type `T`. (not exported)

# Examples

```jldoctest
julia> BSplines.StandardBasisVector(5, 3)
5-element BSplines.StandardBasisVector{Bool}:
 0
 0
 1
 0
 0

julia> BSplines.StandardBasisVector(Float64, 6, 2)
6-element BSplines.StandardBasisVector{Float64}:
 0.0
 1.0
 0.0
 0.0
 0.0
 0.0
```
"""
@propagate_inbounds StandardBasisVector(length, index) = StandardBasisVector(Bool, length, index)
@propagate_inbounds StandardBasisVector(T::Type, length, index) =
    StandardBasisVector{T}(length, index)

Base.size(A::StandardBasisVector) = (A.length,)

Base.copy(A::StandardBasisVector) = A

@inline @propagate_inbounds function Base.getindex(A::StandardBasisVector, i::Int)
    @boundscheck checkbounds(A, i)
    convert(eltype(A), i == A.index)
end
Base.getindex(A::StandardBasisVector, ::Colon) = A

Base.:(==)(x::StandardBasisVector, y::StandardBasisVector) =
    (x.length == y.length) & (x.index == y.index)


################################################################################
###  Introduction to Scientific Programming and Machine Learning with Julia  ###
###                                                                          ###
### Run each script on a new clean Julia session                             ###
### GitHub: https://github.com/sylvaticus/IntroSPMLJuliaCourse               ###
### Licence (apply to all material of the course: scripts, videos, quizes,..)###
### Creative Commons By Attribution (CC BY 4.0), Antonello Lobianco          ###
################################################################################

# ## 0104 Control Flow and Functions

# ## Some stuff to set-up the environment..
cd(@__DIR__)
using Pkg
Pkg.activate(".")
## If using a Julia version different than 1.7 please uncomment and run the following line (reproductibility guarantee will hower be lost)
## Pkg.resolve()
## Pkg.instantiate() # run this if you didn't in Segment 01.01
using Random
Random.seed!(123)
using InteractiveUtils # loaded automatically when working... interactively

# ## Variables scope

# The scope of a variable is the region of code where the variable can be accessed directly (without using prefixes).
# Modules, functions, `for` and other blocks (but notably not "if" blocks) introduce an inner scope that hinerit from the scope where the block or function is defined (but not, for function, from the caller's scope).
# Variables that are defined outside any block or function are _global_ for the module where they are defined (the `Main` module if outside any other module, e.g. on the REPL), the others being _local_.
# Variables defined in a `for` block that already exists as global behave differently depending if we are working interactively or not:

g  = 2
g2 = 20
for i in 1:2
    l1 = g2                                       # l1: local, g2: global (read only)
    l1 += i
    g = i                                         # REPL/INTERACTIVE: global (from Julia 1.5), FILE MODE: local by default (with a warning `g` being already defined)
    g += i
    println("i: $i")
    println("l1: $l1")
    println("g: $g")
    for j in 1:2
        l1 += j                                   # still the local in outer loop, not a new local one
        l2 = j
        g  += j
        println("j: $j")
        println("l1 inside inner loop: $l1")
        println("l2 inside inner loop: $l2")
        println("g inside inner loop: $g")
    end
    ## println("l2 post: $l2")                     # error: l2 not defined in this scope
    println("l1 post: $l1")
    println("g post: $g")
end
## println("l1 global $l1")                        # error; l1 is not defined in the global scope
println("g in global: $g")                        # REPL/INTERACTIVE: "7", FILE MODE: "2"

function foo(i)
    l1 = g2                                       # l1: local, g2: global (read only)
    l1 += i
    g = i                                         # REPL/INTERACTIVE and FILE MODE: local by default (with a warning `g` being already defined)
    g += i
    println("i: $i")
    println("l1: $l1")
    println("g: $g")
    for j in 1:2
        l1 += j                                   # still the local in outer loop, not a new local one
        l2 = j
        g  += j
        println("j: $j")
        println("l1 inside inner loop: $l1")
        println("l2 inside inner loop: $l2")
        println("g inside inner loop: $g")
    end
    ## println("l2 post: $l2")                     # error: l2 not defined in this scope
    println("l1 post: $l1")
    println("g post: $g")
end

println("Calling foo..")
foo(10)
println("g in global: $g")                        # REPL/INTERACTIVE: "7", FILE MODE: "2"

g = 2
include("010401-varScopeExample.jl.txt")            # gives a warning !

# ## Repeated iterations: `for` and `while` loops, List Comprehension, Maps

for i in 1:2, j in 3:4             # j is the inner loop
    println("i: $i, j: $j")
end
a = 1
while true             # or condition, e.g. while a == 10
    global a += 1
    println("a: $a")
    if a == 10
        break
    else
        continue
    end
    println("This is never printed")
end

# ### List Comprehension
[ i+j for i in 1:2, j in 3:4 if j >= 4]

# ### Maps

# Apply a (possible anonymous) function to a list of arguments:

map((name,year) -> println("$name is $year year old"), ["Marc","Anna"], [25,22])

# !!! warninng
#     Don't confuse the single-line arrows used in anonymous functions (`->`) with the double-line arrow used to define a Pair (`=>`)

# We can use maps to substitute values of an array based on a dictionary:
countries = ["US","UK","IT","UK","UK"]
countryNames = Dict("IT" => "Italy", "UK" => "United Kngdom",  "US"=>"United States")
countryLongNames = map(cShortName -> countryNames[cShortName], countries)

# ## Conditional statements: if blocks and ternary operators

a = 10

if a < 4                      # use `!`, `&&` and `||` for "not", "and" and  "or" conditions
    println("a < 4")
elseif a < 8
    println("a < 8")
else
    println("a is big!")
end

a = 10

if a < 5
  b = 100
else
  b = 200
end

b

# Ternary operators
b =  a < 5 ? 100 : 200   # ? condition : if true : if false

# Short-circuit evaluation
b = 100
(a < 5) || (b = 200)    # replace an if: the second part is executed unless the first part is already true
b
(a < 50) || (b = 500)   # here is never executed
b

# !!! warning
#     Don't confuse boolean operators `&&` and `||` with their analogous `&` and `|` bitwise operators

a = 3
b = 2
bitstring(a)
bitstring(b)

##a && b     # error non boolean used in boolean context
a & b
a | b

# ## Functions

function foo(x)             # function definition
    x+2
end
foo(2)                      # function call
inlineFunction(x) = x+2
foo2 = x -> x+2             # anonymous function (aka "lambda function") and assignment to the variable foo2
foo2(2)

# A nested function:
function f1(x)
    function f2(x,y)
        x+y
    end
    f2(x,2)
end

f1(2)

# A recursive function:
function fib(n)                   # This is a naive implementation. Much faster implementations of the Fibonacci numbers exist
    if n == 0  return 0
    elseif n == 1 return 1
    else
     return fib(n-1) + fib(n-2)
    end
end
fib(4)

# ### Function arguments

# #### Positional vs keyword arguments
f(a,b=1;c=1) = a+10b+100c  # `a` and `b` are positional arguments (`b` with a default provided), `c` is a keyword argument
f(2)
f(2,c=3)

foo(a, args...;c=1) = a + length(args) + sum(args) + c
foo(1,2,3,c=4)

# Rules for positional and keyword arguments:
# - keyword arguments follow a semicolon `;` in the parameters list of the function definition
# - a positional argument without a default can not follow a positional argument with a default provided
# - the splat operator to define variable number of arguments must be the last positional argument
# - the function call must use positional arguments by position and keyword arguments by name

## foo(a::String="aaa",b::Int64) = "$a "+string(b) # error! Optional positional argument before a mandatory positional one

# ### Argument types and multiple dispatch

# Simple to understand the usage, complex to understand the deep implications
foo3(a::Int64,b::String) = a + parse(Int64,b)
foo3(2,"3")
foo3(a::String,b::Int64) = parse(Int64,a) + b
foo3("3",2)
methods(foo3)

# Multiple dispatch allows to compile a _specialised_ version JIT at run time, on the first call with the given parameters type
# We will see it again when dealing with type inheritance
# In general, unless we need to write specialised methods, no need to specify the type of the parameters. No influence on performances, this is automatically inferred (and the funciton compiled) based on the run-time type of the argument

# !!! tip Functions performances tip
#     The most important things for performances are (1) that the function is _type stable_, that is, that conditional to a specific combination of the types of the parameters the function returns the same type. This is a condition necessary to have a working chain of type inference across function calls; (2) that no (non constant) global constants are used in the function and indeed all the required information for the functio ndoing its work is embedded in the function parameters

# ### Function templates
foo3(a::T,b::String) where {T<: Number} = a + parse(T,b)             # can use T in the function body
foo3(2,"1")
foo3(1.5,"1.5")
foo4(a::Int64,b::T where T <: Number) = a + b                        # ok not used in functio nbody
foo4(a::Int64,b::Array{T} where T <: Number) = a .+ fill(T,b,2)      # wil lerror, can't use T in the function body
## foo4(2,[1,2])                                                     # run time error, T not defined

# ### Call by reference vs. call by value

# How the variable used as function argument within the function body relates to the variable used in calling the function ?
# - **call by value**: the value of the argument is copied and the function body works on a copy of the value
# - **call by reference**: the function works on the same object being referenced by the caller variable and the function argument
# - **call by sharing** (Julia): the arguments are just new local variables that bind the same object. The effects of "modifications" on the local variable on the caller's one depends on the mutability property of the object as we saw in the _Types and objects_ segment:
#   - immutable objects: we can only have that the argument is rebinded to other objects. No effects on the original caller object
#   - mutable objects: if the argument is rebinded to an other object, no effects on the caller object. If the object is modified, the caller object (being the same object) is also modified

x = 10
foo(y) = y = 1
foo(x)
x
foo(x) = x[1] = 10
x = [1,2]
foo(x)
x
foo3(x) = x = [10,20]
foo3(x)
x

# !!! info
#     Functions that modify at least one of their arguments are named, by convention, with an exclamation mark at the end of their name and the argument(s) that is (are) modified set as the first(s) argument(s)

foo!(x) = x[1] = 10 # to follow the convention

# ## `do` blocks

# Functions that accept an other function as their first parameter can be rewritten with the function itself defined in a `do` block:
using Statistics
pool(f,x,poolSize=3) = [f(x[i:i+poolSize-1]) for i in 1:length(x)-poolSize+1] # a real case, used in neural networks as pooling layer
pool(mean,[1,2,3,4,5,6])
pool(maximum,[1,2,3,4,5,6])
pool([1,2,3,4,5]) do x      # x is a local variable within the do block. We need as many local variables as the number of parameters of the inner function
    sum(x)/length(x)
end
# Using the `do` block we can call the outer function and define the inner function at the same time.
# `do` blocks are frequently used in input/output operations


"""
    sign_cadlag(x::N) where {N<:Real}

This function works like the sign function but is ``1`` for input ``0``.

### Input

- `x` -- real scalar

### Output

``1`` if ``x ≥ 0``, ``-1`` otherwise.

### Notes

This is the sign function right-continuous at zero (see
[càdlàg function](https://en.wikipedia.org/wiki/C%C3%A0dl%C3%A0g)).
It can be used with vector-valued arguments via the dot operator.

### Examples

```jldoctest
julia> LazySets.sign_cadlag.([-0.6, 1.3, 0.0])
3-element Array{Float64,1}:
 -1.0
  1.0
  1.0
```
"""
function sign_cadlag(x::N) where {N<:Real}
    return x < zero(x) ? -one(x) : one(x)
end

"""
    substitute(substitution::Dict{Int, T}, x::AbstractVector{T}) where {T}

Apply a substitution to a given vector.

### Input

- `substitution` -- substitution (a mapping from an index to a new value)
- `x`            -- vector

### Output

A fresh vector corresponding to `x` after `substitution` was applied.
"""
function substitute(substitution::Dict{Int, T}, x::AbstractVector{T}) where {T}
    return substitute!(substitution, copy(x))
end

"""
    substitute!(substitution::Dict{Int, T}, x::AbstractVector{T}) where {T}

Apply a substitution to a given vector.

### Input

- `substitution` -- substitution (a mapping from an index to a new value)
- `x`            -- vector (modified in this function)

### Output

The same (but see the Notes below) vector `x` but after `substitution` was
applied.

### Notes

The vector `x` is modified in-place if it has type `Vector` or `SparseVector`.
Otherwise, we first create a new `Vector` from it.
"""
function substitute!(substitution::Dict{Int, T}, x::AbstractVector{T}) where {T}
    return substitute!(Vector(x), substitution)
end

function substitute!(substitution::Dict{Int, T},
                     x::Union{Vector{T}, SparseVector{T}}) where {T}
    for (index, value) in substitution
        x[index] = value
    end
    return x
end

"""
    reseed(rng::AbstractRNG, seed::Union{Int, Nothing})

Reset the RNG seed if the seed argument is a number.

### Input

- `rng`  -- random number generator
- `seed` -- seed for reseeding

### Output

The input RNG if the seed is `nothing`, and a reseeded RNG otherwise.
"""
function reseed(rng::AbstractRNG, seed::Union{Int, Nothing})
    if seed != nothing
        return Random.seed!(rng, seed)
    end
    return rng
end

"""
    StrictlyIncreasingIndices

Iterator over the vectors of `m` strictly increasing indices from 1 to `n`.

### Fields

- `n` -- size of the index domain
- `m` -- number of indices to choose (resp. length of the vectors)

### Notes

The vectors are modified in-place.

The iterator ranges over ``\\binom{n}{m}`` (`n` choose `m`) possible vectors.

This implementation results in a lexicographic order with the last index growing
first.

### Examples

```jldoctest
julia> for v in LazySets.StrictlyIncreasingIndices(4, 2)
           println(v)
       end
[1, 2]
[1, 3]
[1, 4]
[2, 3]
[2, 4]
[3, 4]
```
"""
struct StrictlyIncreasingIndices
    n::Int
    m::Int

    function StrictlyIncreasingIndices(n::Int, m::Int)
        @assert n >= m > 0 "require n >= m > 0"
        new(n, m)
    end
end

Base.eltype(::Type{StrictlyIncreasingIndices}) = Vector{Int}
Base.length(sii::StrictlyIncreasingIndices) = binomial(sii.n, sii.m)

# initialization
function Base.iterate(sii::StrictlyIncreasingIndices)
    v = [1:sii.m;]
    return (v, v)
end

# normal iteration
function Base.iterate(sii::StrictlyIncreasingIndices, state::AbstractVector{Int})
    v = state
    i = sii.m
    diff = sii.n
    if i == diff
        return nothing
    end
    while v[i] == diff
        i -= 1
        diff -= 1
    end
    # update vector
    v[i] += 1
    for j in i+1:sii.m
        v[j] = v[j-1] + 1
    end
    # detect termination: first index has maximum value
    if i == 1 && v[1] == (sii.n - sii.m + 1)
        return (v, nothing)
    end
    return (v, v)
end

# termination
function Base.iterate(sii::StrictlyIncreasingIndices, state::Nothing)
    return nothing
end

"""
    subtypes(interface, concrete::Bool)

Return the concrete subtypes of a given interface.

### Input

- `interface` -- an abstract type, usually a set interface
- `concrete`  -- if `true`, seek further the inner abstract subtypes of the given
                 interface, otherwise return only the direct subtypes of `interface`

### Output

A list with the subtypes of the abstract type `interface`, sorted alphabetically.

### Examples

Consider the `AbstractPolytope` interface. If we include the abstract subtypes
of this interface,

```jldoctest subtypes
julia> using LazySets: subtypes

julia> subtypes(AbstractPolytope, false)
4-element Array{Any,1}:
 AbstractCentrallySymmetricPolytope
 AbstractPolygon
 HPolytope
 VPolytope
```

We can use this function to obtain the concrete subtypes of
`AbstractCentrallySymmetricPolytope` and `AbstractPolygon` (further until all
concrete types are obtained), using the `concrete` flag:

```jldoctest subtypes
julia> subtypes(AbstractPolytope, true)
14-element Array{Type,1}:
 Ball1
 BallInf
 HPolygon
 HPolygonOpt
 HPolytope
 Hyperrectangle
 Interval
 LineSegment
 Singleton
 SymmetricIntervalHull
 VPolygon
 VPolytope
 ZeroSet
 Zonotope
```
"""
function subtypes(interface, concrete::Bool)

    subtypes_to_test = subtypes(interface)

    # do not seek the concrete subtypes further
    if !concrete
        return sort(subtypes_to_test, by=string)
    end

    result = Vector{Type}()
    i = 0
    while i < length(subtypes_to_test)
        i += 1
        subtype = subtypes_to_test[i]
        new_subtypes = subtypes(subtype)
        if isempty(new_subtypes)
            # base type found
            push!(result, subtype)
        else
            # yet another interface layer
            append!(subtypes_to_test, new_subtypes)
        end
    end
    return sort(result, by=string)
end

"""
    implementing_sets(op::Function;
                      signature::Tuple{Vector{Type}, Int}=(Type[], 1),
                      type_args=Float64, binary::Bool=false)

Compute a dictionary containing information about availability of (unary or
binary) concrete set operations.

### Input

- `op`        -- set operation (respectively its `Function` object)
- `signature` -- (optional, default: `Type[]`) the type signature of the
                 function without the `LazySet` type(s) (see also the `index`
                 option and the `Examples` section below)
- `index`     -- (optional, default: `1`) index of the set type in the signature
                 in the unary case (see the `binary` option)
- `type_args` -- (optional, default: `Float64`) type arguments added to the
                 `LazySet`(s) when searching for available methods; valid
                 inputs are a type or `nothing`, and in the unary case (see the
                 `binary` option) it can also be a list of types
- `binary`    -- (optional, default: `false`) flag indicating whether `op` is a
                 binary function (`true`) or a unary function (`false`)

### Output

A dictionary with three keys each mapping to a list:
- `"available"` -- This list contains all set types such that there exists an
                   implementation of `op`.
- `"missing"`   -- This list contains all set types such that there does not
                   exist an implementation of `op`. Note that this is the
                   complement of the `"available"` list.
- `"specific"`  -- This list contains all set types such that there exists a
                   type-specific implementation. Note that those set types also
                   occur in the `"available"` list.

In the unary case, the lists contain set types.
In the binary case, the lists contain pairs of set types.

### Examples

```jldoctests
julia> using LazySets: implementing_sets

julia> dict = implementing_sets(tovrep);

julia> dict["available"]  # tovrep is only available for polyhedral set types
6-element Array{Type,1}:
 HPolygon
 HPolygonOpt
 HPolyhedron
 HPolytope
 VPolygon
 VPolytope

julia> dict = implementing_sets(σ; signature=Type[AbstractVector{Float64}], index=2);

julia> dict["missing"]  # every set type implements function σ
0-element Array{Type,1}

julia> N = Rational{Int};  # restriction of the number type

julia> dict = implementing_sets(σ; signature=Type[AbstractVector{N}], index=2, type_args=N);

julia> dict["missing"]  # some set types are not available with number type N
4-element Array{Type,1}:
 Ball2
 Ballp
 Bloating
 Ellipsoid

julia> dict = LazySets.implementing_sets(convex_hull; binary=true);  # binary case

julia> (HPolytope, HPolytope) ∈ dict["available"]  # dict contains pairs now
true
```
"""
function implementing_sets(op::Function;
                           signature::Vector{Type}=Type[],
                           index::Int=1,
                           type_args=Float64,
                           binary::Bool=false)
    function get_unary_dict()
        return Dict{String, Vector{Type}}("available" => Vector{Type}(),
                                          "missing" => Vector{Type}(),
                                          "specific" => Vector{Type}())
    end

    if !binary
        # unary case
        dict = get_unary_dict()
        _implementing_sets_unary!(dict, op, signature, index, type_args)
        return dict
    end

    # binary case by considering all set types as second argument
    binary_dict = Dict{String, Vector{Tuple{Type, Type}}}(
        "available" => Vector{Tuple{Type, Type}}(),
        "missing" => Vector{Tuple{Type, Type}}(),
        "specific" => Vector{Tuple{Type, Type}}())
    signature_copy = copy(signature)
    insert!(signature_copy, 1, LazySet)  # insert a dummy type
    for set_type in subtypes(LazySet, true)
        augmented_set_type = set_type
        try
            if type_args isa Type
                augmented_set_type = set_type{type_args}
            else
                @assert type_args == nothing "for binary functions, " *
                    "`type_args` must not be a list"
            end
        catch e
            # type combination not available
            push!(binary_dict["missing"], (set_type, LazySet))
            continue
        end
        signature_copy[1] = augmented_set_type  # replace dummy type
        unary_dict = get_unary_dict()
        _implementing_sets_unary!(unary_dict, op, signature_copy, 1, type_args)

        # create pairs
        for key in ["available", "missing", "specific"]
            i = 1
            for other_set_type in unary_dict[key]
                push!(binary_dict[key], (set_type, other_set_type))
                i += 1
            end
        end
    end
    return binary_dict
end

function _implementing_sets_unary!(dict, op, signature, index, type_args)
    function create_type_tuple(given_type)
        if isempty(signature)
            return Tuple{given_type}
        end
        # create dynamic tuple
        new_signature = copy(signature)
        insert!(new_signature, index, given_type)
        return Tuple{new_signature...}
    end

    for set_type in subtypes(LazySet, true)
        augmented_set_type = set_type
        try
            if type_args isa Type
                augmented_set_type = set_type{type_args}
            elseif type_args != nothing
                augmented_set_type = set_type{type_args...}
            end
        catch e
            # type combination not available
            push!(dict["missing"], set_type)
            continue
        end
        tuple_set_type = create_type_tuple(augmented_set_type)
        if !hasmethod(op, tuple_set_type)
            # implementation not available
            push!(dict["missing"], set_type)
            continue
        end
        # implementation available
        push!(dict["available"], set_type)

        # check if there is a type-specific method
        super_type = supertype(augmented_set_type)
        tuple_super_type = create_type_tuple(super_type)
        if !hasmethod(op, tuple_super_type)
            continue
        end
        method_set_type = which(op, tuple_set_type)
        method_super_type = which(op, tuple_super_type)
        if method_set_type != method_super_type
            push!(dict["specific"], set_type)
        end
    end
end


# This file is a part of AstroLib.jl. License is MIT "Expat".
# Copyright (C) 2016 Mosè Giordano.

"""
    daycnv(julian_days) -> DateTime

### Purpose ###

Converts Julian days number to Gregorian calendar dates.

### Explanation ###

Takes the number of Julian calendar days since epoch `-4713-11-24T12:00:00` and
returns the corresponding proleptic Gregorian Calendar date.

### Argument ###

* `julian_days`: Julian days number.

### Output ###

Proleptic Gregorian Calendar date, of type `DateTime`, corresponding to the
given Julian days number.

### Example ###

```jldoctest
julia> using AstroLib

julia> daycnv(2440000)
1968-05-23T12:00:00
```

### Notes ###

`jdcnv` is the inverse of this function.
"""
const daycnv=Dates.julian2datetime


"""
    neighbors(vec::AbstractVector)

Returns an iterator over neighboring pairs.
```julia
julia> neighbors([1,2,3]) |> collect
2-element Array{Tuple{Int64,Int64},1}:
 (1, 2)
 (2, 3)
 ```
 """
function neighbors(itr::AbstractVector)
    first  = @view itr[1:end-1]
    second = @view itr[2:end]
    zip(first, second)
end


abstract type SaturationGoal end

reached(g::EGraph, goal::Nothing) = false
reached(g::EGraph, goal::SaturationGoal) = false

"""
This goal is reached when the `exprs` list of expressions are in the
same equivalence class.
"""
struct EqualityGoal <: SaturationGoal
    exprs::Vector{Any}
    ids::Vector{EClassId}
    function EqualityGoal(exprs, eclasses)
        @assert length(exprs) == length(eclasses) && length(exprs) != 0
        new(exprs, eclasses)
    end
end

function reached(g::EGraph, goal::EqualityGoal)
    all(x -> in_same_class(g, goal.ids[1], x), @view goal.ids[2:end])
end

"""
Boolean valued function as an arbitrary saturation goal.
User supplied function must take an [`EGraph`](@ref) as the only parameter.
"""
struct FunctionGoal <: SaturationGoal
    fun::Function
end

function reached(g::EGraph, goal::FunctionGoal)::Bool
    fun(g)
end

mutable struct Report
    reason::Union{Symbol, Nothing}
    egraph::EGraph
    iterations::Int
    to::TimerOutput
end

Report() = Report(nothing, EGraph(), 0, TimerOutput())
Report(g::EGraph) = Report(nothing, g, 0, TimerOutput())

# string representation of timedata
function Base.show(io::IO, x::Report)
    g = x.egraph
    println(io, "Equality Saturation Report")
    println(io, "=================")
    println(io, "\tStop Reason: $(x.reason)")
    println(io, "\tIterations: $(x.iterations)")
    # println(io, "\tRules applied: $(g.age)")
    println(io, "\tEGraph Size: $(g.numclasses) eclasses, $(length(g.memo)) nodes")
    print_timer(io, x.to)
end

"""
Configurable Parameters for the equality saturation process.
"""
@with_kw mutable struct SaturationParams
    timeout::Int = 8
    timelimit::Period = Second(-1)
    # default sizeout. TODO make this bytes
    # sizeout::Int = 2^14
    matchlimit::Int = 5000
    eclasslimit::Int = 5000
    enodelimit::Int = 15000
    goal::Union{Nothing, SaturationGoal} = nothing
    stopwhen::Function = ()->false
    scheduler::Type{<:AbstractScheduler} = BackoffScheduler
    schedulerparams::Tuple=()
    threaded::Bool = false
    timer::Bool = true
    printiter::Bool = false
    simterm::Function = similarterm
end

struct Match
    rule::AbstractRule
    # the rhs pattern to instantiate
    pat_to_inst
    # the substitution
    sub::Sub
    # the id the matched the lhs
    id::EClassId
end

const MatchesBuf = Vector{Match}

function cached_ids(g::EGraph, p::AbstractPat)# ::Vector{Int64}
    if isground(p)
        id = lookup_pat(g, p)
        !isnothing(id) && return [id]
    else
        return collect(keys(g.classes))
    end
    return []
end

function cached_ids(g::EGraph, p) # p is a literal
    id = lookup(g, ENodeLiteral(p))
    !isnothing(id) && return [id]
    return []
end

# FIXME
function cached_ids(g::EGraph, p::PatTerm)
    # println("pattern $p, $(p.head)")
    # println("all ids")
    # keys(g.classes) |> println
    # println("cached symbols")
    # cached = get(g.symcache, p.head, Set{Int64}())
    # println("symbols where $(p.head) appears")
    # appears = Set{Int64}()
    # for (id, class) ∈ g.classes
    #     for n ∈ class
    #         if n.head == p.head
    #             push!(appears, id)
    #         end
    #     end
    # end
    # # println(appears)
    # if !(cached == appears)
    #     @show cached
    #     @show appears
    # end

    collect(keys(g.classes))
    # cached
    # get(g.symcache, p.head, [])
end

# function cached_ids(g::EGraph, p::PatLiteral)
#     get(g.symcache, p.val, [])
# end

function (r::SymbolicRule)(g::EGraph, id::EClassId)
    ematch(g, r.ematch_program, id) .|> sub -> Match(r, r.right, sub, id)
end

function (r::DynamicRule)(g::EGraph, id::EClassId)
    ematch(g, r.ematch_program, id) .|> sub -> Match(r, nothing, sub, id)
end

function (r::BidirRule)(g::EGraph, id::EClassId)
    vcat(ematch(g, r.ematch_program_l, id) .|> sub -> Match(r, r.right, sub, id),
        ematch(g, r.ematch_program_r, id) .|> sub -> Match(r, r.left, sub, id))
end

"""
Returns an iterator of `Match`es.
"""
function eqsat_search!(egraph::EGraph, theory::Vector{<:AbstractRule},
    scheduler::AbstractScheduler, report; threaded=false)
    match_groups = Vector{Match}[]
    function pmap(f, xs)
        # const propagation should be able to optimze one of the branch away
        if threaded
            # # try to divide the work evenly between threads without adding much overhead
            # chunks = Threads.nthreads() * 10
            # basesize = max(length(xs) ÷ chunks, 1)
            # ThreadsX.mapi(f, xs; basesize=basesize)
            ThreadsX.map(f, xs)
        else
            map(f, xs)
        end
    end

    inequalities = filter(Base.Fix2(isa, UnequalRule), theory)
    # never skip contradiction checks
    append_time = TimerOutput()
    for rule ∈ inequalities
        @timeit report.to repr(rule) begin
            ids = cached_ids(egraph, rule.left)
            rule_matches = pmap(i -> rule(egraph, i), ids)
            @timeit append_time "appending matches" begin
                append!(match_groups, rule_matches)
            end
        end
    end

    other_rules = filter(theory) do rule
        !(rule isa UnequalRule)
    end
    for rule ∈ other_rules
        @timeit report.to repr(rule) begin
            # don't apply banned rules
            if !cansearch(scheduler, rule)
                # println("skipping banned rule $rule")
                continue
            end
            ids = cached_ids(egraph, rule.left)
            rule_matches = pmap(i -> rule(egraph, i), ids)

            n_matches = isempty(rule_matches) ? 0 : sum(length, rule_matches)
            # @show (rule, n_matches)
            can_yield = inform!(scheduler, rule, n_matches)
            if can_yield
                @timeit append_time "appending matches" begin
                    append!(match_groups, rule_matches)
                end
            end
        end
    end

    # @timeit append_time "appending matches" begin
    #     result = reduce(vcat, match_groups) # this should be more efficient than multiple appends
    # end
    merge!(report.to, append_time, tree_point=["Search"])

    return Iterators.flatten(match_groups)
    # return result
end

function (rule::UnequalRule)(g::EGraph, match::Match; simterm=similarterm)
    lc = match.id
    rinst = instantiate(g, match.pat_to_inst, match.sub, rule; simterm=simterm)
    rc, node = addexpr!(g, rinst)

    if find(g, lc) == find(g, rc)
        @log "Contradiction!" rule
        return :contradiction
    end
    return nothing
end

function (rule::SymbolicRule)(g::EGraph, match::Match; simterm=similarterm)
    rinst = instantiate(g, match.pat_to_inst, match.sub, rule; simterm=simterm)
    rc, node = addexpr!(g, rinst)
    merge!(g, match.id, rc.id)
    return nothing
end

function (rule::DynamicRule)(g::EGraph, match::Match; simterm=similarterm)
    f = rule.rhs_fun
    actual_params = [instantiate(g, PatVar(v, i, alwaystrue), match.sub, rule) for (i, v) in enumerate(rule.patvars)]
    r = f(g[match.id], match.sub, g, actual_params...)
    isnothing(r) && return nothing
    rc, node = addexpr!(g, r)
    merge!(g, match.id, rc.id)
    return nothing
end

function eqsat_apply!(g::EGraph, matches, rep::Report, params::SaturationParams)
    i = 0
    # println.(matches)
    for match ∈ matches
        i += 1

        # if params.eclasslimit > 0 && g.numclasses > params.eclasslimit
        #     @log "E-GRAPH SIZEOUT"
        #     rep.reason = :eclasslimit
        #     return
        # end

        if reached(g, params.goal)
            @log "Goal reached"
            rep.reason = :goalreached
            return
        end

        rule = match.rule
        # println("applying $rule")

        halt_reason = rule(g, match; simterm=params.simterm)
        if (halt_reason !== nothing)
            rep.reason = halt_reason
            return
        end

        # println(rule)
        # println(sub)
        # println(l); println(r)
        # display(egraph.classes); println()
    end
end

import ..@log

"""
Core algorithm of the library: the equality saturation step.
"""
function eqsat_step!(g::EGraph, theory::Vector{<:AbstractRule}, curr_iter,
        scheduler::AbstractScheduler, match_hist::MatchesBuf,
        params::SaturationParams, report)

    instcache = Dict{AbstractRule, Dict{Sub, EClassId}}()

    setiter!(scheduler, curr_iter)

    matches = @timeit report.to "Search" eqsat_search!(g, theory, scheduler, report; threaded=params.threaded)

    # matches = setdiff!(matches, match_hist)

    @timeit report.to "Apply" eqsat_apply!(g, matches, report, params)

    # union!(match_hist, matches)

    if report.reason === nothing && cansaturate(scheduler) && isempty(g.dirty)
        report.reason = :saturated
    end
    @timeit report.to "Rebuild" rebuild!(g)

    return report, g
end

"""
Given an [`EGraph`](@ref) and a collection of rewrite rules,
execute the equality saturation algorithm.
"""
function saturate!(g::EGraph, theory::Vector{<:AbstractRule}, params=SaturationParams())
    curr_iter = 0

    sched = params.scheduler(g, theory, params.schedulerparams...)
    match_hist = MatchesBuf()
    report = Report(g)

    start_time = Dates.now().instant

    !params.timer && disable_timer!(report.to)
    timelimit = params.timelimit > Second(0)

    while true
        curr_iter+=1

        params.printiter && @info("iteration ", curr_iter)

        report, egraph = eqsat_step!(g, theory, curr_iter, sched, match_hist, params, report)

        elapsed = Dates.now().instant - start_time

        if timelimit && params.timelimit <= elapsed
            report.reason = :timelimit
            break
        end

        # report.reason == :matchlimit && break
        if !(report.reason isa Nothing)
            break
        end

        if curr_iter >= params.timeout
            report.reason = :timeout
            break
        end

        if params.eclasslimit > 0 && g.numclasses > params.eclasslimit
            # println(params.eclasslimit)
            report.reason = :eclasslimit
            break
        end

        if reached(g, params.goal)
            report.reason = :goalreached
            break
        end
    end
    report.iterations = curr_iter
    @log report

    return report
end

function areequal(theory::Vector, exprs...; params=SaturationParams())
    g = EGraph(exprs[1])
    areequal(g, theory, exprs...; params=params)
end

function areequal(g::EGraph, t::Vector{<:AbstractRule}, exprs...; params=SaturationParams())
    @log "Checking equality for " exprs
    if length(exprs) == 1; return true end
    # rebuild!(G)

    @log "starting saturation"

    n = length(exprs)
    ids = Vector{EClassId}(undef, n)
    nodes = Vector{AbstractENode}(undef, n)
    for i ∈ 1:n
        ec, node = addexpr!(g, exprs[i])
        ids[i] = ec.id
        nodes[i] = node
    end

    goal = EqualityGoal(collect(exprs), ids)

    # alleq = () -> (all(x -> in_same_set(G.uf, ids[1], x), ids[2:end]))

    params.goal = goal
    # params.stopwhen = alleq

    report = saturate!(g, t, params)

    # display(g.classes); println()
    if !(report.reason === :saturated) && !reached(g, goal)
        return missing # failed to prove
    end
    return reached(g, goal)
end

macro areequal(theory, exprs...)
    esc(:(areequal($theory, $exprs...)))
end

macro areequalg(G, theory, exprs...)
    esc(:(areequal($G, $theory, $exprs...)))
end