text:
\section{Convex Optimization}
\subsection{Convexity}
\subsubsection{Convex Sets}
A set $C$ is convex if, for any $x,y \in C$ and $\alpha \in \mathbb{R}$ with
$0 \leq \alpha \leq 1$, we have
\begin{equation}
  \alpha x + (1-\alpha) y \in C
\end{equation}
This point is known as a convex combination of $x$ and $y$.
\subsubsection{Convex Functions}
A function
$f: \mathbb{R}^n \to \mathbb{R}$ is convex if its domain $\mathcal{D}(f)$ is
a convex set, and if, for all $x, y \in \mathcal{D}(f)$ and
$\alpha \in \mathbb{R}$, $0 \leq \alpha \leq 1$, we have:
\begin{equation}
  f(\alpha x + (1-\alpha) y) \leq \alpha f(x) + (1-\alpha) f(y)
\end{equation}
A function is strictly convex if:
\begin{equation}
  f(\alpha x + (1-\alpha) y) < \alpha f(x) + (1-\alpha) f(y)
\end{equation}
\subsubsection{First-Order Conditions}
Suppose a function $f: \mathbb{R}^n \to \mathbb{R}$ is differentiable, then
$f$ is convex if and only if $\mathcal{D}(f)$ is a convex set and for all
$x, y \in \mathcal{D}(f)$ we have:
\begin{equation}
  f(y) \geq f(x) + \nabla_xf(x)^T(y-x)
\end{equation}
This is called the first-order approximation to the function $f$ at point $x$.
This is approximating $f$ with its tangent line at point $x$.
The first order condition for convexity says that $f$ is convex if and only if
the tangent line is a global underestimator of the function $f$.
In other words, if we take our function and draw a tangent line at any point,
then every point on this line will lie below the corresponding point on $f$.
\subsubsection{Second-Order Conditions}
Suppose a function $f: \mathbb{R}^n \to \mathbb{R}$ is twice differentiable,
i.e. the hessian $\nabla^2_xf(x)$ is defined for all points $x$ in the domain
of $f$. Then $f$ is convex if and only if $\mathcal{D}(f)$ is a convex set and
its hessian is PSD:
\begin{equation}
  \nabla^2_xf(y) \succeq 0
\end{equation}
\subsubsection{Jensen's Inequality}
Suppose we start with the inequality in the basic definition of a convex function:
\begin{equation}
  f(\alpha x + (1-\alpha) y) \leq \alpha f(x) + (1-\alpha) f(y)
\end{equation}
We can extend this to convex combinations of more than one point:
\begin{equation}
  f\left(\sum_{i=1}^k \alpha_i x_i\right) \leq \sum^k_{i=1} \alpha_i f(x_i)
\end{equation}
For $\sum_k \alpha = 1$ and all $\alpha \geq 0$. This can be extended to
integrals:
\begin{equation}
  f \left( \int p ( x ) x d x \right) \leq \int p ( x ) f ( x ) dx
\end{equation}
For $\int p(x)dx = 1$ and $p(x) \geq 0 \forall x$. This implies $p(x)$ is a
probability density, and we can write our inequality in terms of expectations:
\begin{equation}
  f ( \mathbf { E } [ x ] ) \leq \mathbf { E } [ f ( x ) ]
\end{equation}
And is known as Jensen's inequality.
\subsubsection{Sublevel Sets}
Given a convex function $f:\mathbb{R}^n \to \mathbb{R}$ and a real number
$\alpha \in \mathbb{R}$, the $\alpha$-sublevel set is:
\begin{equation}
  \{x \in \mathcal{D}(f): f(x) \leq \alpha\}
\end{equation}
In other words, the $\alpha$-sublevel set is the set of all points $x$
such that $f(x) \leq \alpha$.
\subsection{Convex Optimization}
A convex optimization problem seeks to optimize a convex function $f$ with
respect to a variable $x$ and possible constraints.
\begin{equation}
  \begin{array} { l l } { \text { minimize } } & { f ( x ) } \\ { \text { subject to } } & { g _ { i } ( x ) \leq 0 , \quad i = 1 , \ldots , m } \\ { } & { h _ { i } ( x ) = 0 , \quad i = 1 , \ldots , p } \end{array}
\end{equation}
where $f$ is a convex function, $g_i$ are convex functions, and $h_i$ are affine
functions. The optimal value $p^*$ is equal to the minimum possible value of
the objective function in the feasible region:
\begin{equation}
  p ^ { \star } = \min \left\{ f ( x ) : g _ { i } ( x ) \leq 0 , i = 1 , \ldots , m , h _ { i } ( x ) = 0 , i = 1 , \ldots , p \right\}
\end{equation}
The optimal point $x^*$ is a point such that $f(x^*) = p^*$.
\subsubsection{Global Optimality}
A point $x$ is locally optimal if it is feasible and if there exists some
$R > 0$ such that all feasible points $z$ with $||x-z||_2 \leq R$, satisfy
$f(x) \leq f(z)$. This means that for $x$ to be locally optimal, there
exists no nerby points that have a lower objective value. A point $x$ is globally
optimal if it is feadible and for all feasible points $z$, $f(x) \leq f(z)$.
For a convex optimization problem, all locally optimal points are globally
optimal.
\subsubsection{Gradient Descent}
Using $\alpha \in \mathbb{R}$ as the learning rate, we can update a set of
parameters $\theta$ with respect to minimizing a function $f$ as follows:
\begin{equation}
  \theta := \theta-\alpha\nabla f(\theta)
\end{equation}
Stochastic gradient descent (SGD) is updating the parameter based on each
training example, and batch gradient descent is on a batch of training
examples.
\subsubsection{Newton's Algorithm}
Newton's algorithm is a numerical method using information from the second
derivative to find $\theta$ such that $f'(\theta)=0$.
\begin{equation}
  \theta := \theta-\frac{f'(\theta)}{f''(\theta)}
\end{equation}
For multidimensional parameters:
\begin{equation}
  \theta := \theta-\alpha H^{-1} \nabla_\theta f(\theta)
\end{equation}
Where $H$ is the hessian matrix of second partial derivatives.
\begin{equation}
  H_{ij} = \frac { \partial ^ { 2 } f ( \theta ) } { \partial \theta _ { i } \partial \theta _ { j } }
\end{equation}
\subsection{Lagrange Duality and KKT Conditions}
Generic differentiable convex optimization problems are of the form:
\begin{equation}
  \begin{array} { l l } { \text { minimize } } & { f ( x ) } \\ { \text { subject to } } & { g _ { i } ( x ) \leq 0 , \quad i = 1 , \ldots , m } \\ { } & { h _ { i } ( x ) = 0 , \quad i = 1 , \ldots , p } \end{array}
\end{equation}
where $x\in \mathbb{R}^n$ is the optimization variable,
$f: \mathbb{R}^n \to \mathbb{R}$ and $g_i: \mathbb{R}^n \to \mathbb{R}$
are differentiable convex functions, and $h_i: \mathbb{R}^n \to \mathbb{R}$
are affine functions.
\subsubsection{The Lagrangian}
Given a convex constrained minimization problem, the generalized lagrangian
is a function $\mathcal{L}: \mathbb{R}^n \times \mathbb{R}^m \times \mathbb{R}^p
\to \mathbb{R}$ defined as:
\begin{equation}
  \mathcal { L } ( x , \alpha , \beta ) = f ( x ) + \sum _ { i = 1 } ^ { m } \alpha _ { i } g _ { i } ( x ) + \sum _ { i = 1 } ^ { p } \beta _ { i } h _ { i } ( x )
\end{equation}
We refer to $x$ as the primal variables of the Lagrangian. The $\alpha$ and
$\beta$ are refered to as the dual variables or lagrange multipliers. The
key idea behind Lagrangian duality is for any convex optimization problem,
there always exist settings of the dual variables such that the unconstrained
minimum of the Lagrangian with respect to the primal variables (keeping the
dual variables fixed) coincides with the solution of the original constrained
minimization problem.
\subsubsection{Primal and Dual Problems}
The primal problem is:
\begin{equation}
  \min _ { x } \underbrace { \left[ \max _ { \alpha , \beta : \alpha _ { i } \geq 0 , \forall i } \mathcal { L } ( x , \alpha , \beta ) \right] } _ { \theta _ { \mathcal { P } ( x ) } } = \min _ { x } \theta _ { \mathcal { P } } ( x )
\end{equation}
Here, $\theta_{\mathcal{P}} : \mathbb{R}^n \to \mathbb{R}$ is the primal objective, and
the right hand side is the primal problem. In addition,  $p^* = \theta_{\mathcal{P}}(x^*)$
is the optimal value of the primal objective. \\
The dual problem is:
\begin{equation}
  \max _ { \alpha , \beta : \alpha _ { i } \geq 0 , \forall i  } \underbrace { \left[ \min _ { x } \mathcal { L } ( x , \alpha , \beta ) \right] } _ { \theta _ { \mathcal { D } ( x ) } } = \max _ { \alpha , \beta : \alpha _ { i } \geq 0 , \forall i  } \theta _ { \mathcal { D } } ( x )
\end{equation}
Here, $\theta_{\mathcal{D}} : \mathbb{R}^m \times \mathbb{R}^p \to \mathbb{R}$
is the dual objective, and
the right hand side is the dual problem. In addition,
$s^* = \theta_{\mathcal{D}}(\alpha^*, \beta^*)$
is the optimal value of the dual objective.
\subsubsection{Strong and Weak Duality}
Weak duality for any pair of primal and dual problems, with $d^*$ the optimal
objective value of the dual problem and $p^*$ as the optimal objective value
of the primal problem we have:
\begin{equation}
  d^* \leq p^*
\end{equation}
Strong duality is when for any pair of primal and dual problems which satisfy
certain technical conditions called constraint qualifications, then:
\begin{equation}
  d^* = p^*
\end{equation}
\subsubsection{Complementary Slackness}
If strong duality holds, then $\alpha^*_i g(x^*_i) = 0$ for each $i=1,\hdots,m$.
This property is known as complementary slackness. This implies the following:
\begin{equation}
  \alpha _ { i } ^ { * } > 0 \Longrightarrow g _ { i } \left( x ^ { * } \right) = 0
\end{equation}
\begin{equation}
  g _ { i } \left( x ^ { * } \right) < 0 \quad \Longrightarrow \quad \alpha _ { i } ^ { * } = 0
\end{equation}
\subsubsection{The KKT Conditions}
Suppose that $x^* \in \mathbb{R}^n$, $\alpha^* \in \mathbb{R}^m$ and
$\beta^* \in \mathcal{R}^p$ satisfy the following conditions:
\begin{enumerate}
  \item Primal feasibility: $g_i(x^*) \leq 0$, for $i=1,\hdots, m$ and
    $h_i(x^*)=0$ for $i=1,\hdots, p$
  \item Dual feasibility: $\alpha^*_i \geq 0$ for $i=1,\hdots, m$
  \item Complementary slackness: $\alpha^*_i g_i(x^*) = 0$ for $i=1,\hdots, m$
  \item Lagrangian stationarity: $\nabla_x \mathcal{L}(x^*, \alpha^*, \beta^*) = \vec{0}$
\end{enumerate}
Then $x^*$ is primal optimal and $(\alpha^*, \beta^*)$ are dual optimal.
Furthermore, if strong duality holds, then any primal optimal $x^*$ and dual
optimal $(\alpha^*, \beta^*)$ must satisfy the conditions $1$ through $4$.
These conditions are known as the Karush-Kuhn-Tucker (KKT) conditions.

ext: tex

max_stars_repo_name: nextBillyonair/StudyGuide

max_issues_repo_name: nextBillyonair/StudyGuide

max_forks_repo_name: nextBillyonair/StudyGuide

lm_label:
1. YES
2. YES


text:
\lab{Monte Carlo Integration}{Monte Carlo Integration}
\objective{Many important integrals cannot be evaluated symbolically because the integrand has no antiderivative.
Traditional numerical integration techniques like Newton-Cotes formulas and Gaussian quadrature usually work well for one-dimensional integrals, but rapidly become inefficient in higher dimensions.
Monte Carlo integration is an integration strategy that has relatively slow convergence, but that does extremely well in high-dimensional settings compared to other techniques.
In this lab we implement Monte Carlo integration and apply it to a classic problem in statistics.
}

% For example, the integral of the joint normal distribution
% \[
% \int_{\Omega} \frac{1}{\sqrt{(2 \pi)^k}} e^{- \frac{\bold{x}\trp\bold{x}}{2}}
% \]

\section*{Volume Estimation} % ================================================

Since the area of a circle of radius $r$ is $A = \pi r^2$, one way to numerically estimate $\pi$ is to compute the area of the unit circle.
Empirically, we can estimate the area by randomly choosing points in a domain that encompasses the unit circle.
The percentage of points that land within the unit circle approximates the percentage of the area of the domain that the unit circle occupies.
Multiplying this percentage by the total area of the sample domain gives an estimate for the area of the circle.

Since the unit circle has radius $r=1$, consider the square domain $\Omega=[-1,1]\times[-1,1]$.
The following code samples $2000$ uniformly distributed random points in $\Omega$, determines what percentage of those points are within the unit circle, then multiplies that percentage by $4$ (the area of $\Omega$) to get an estimate for $\pi$.

\begin{lstlisting}
>>> import numpy as np
>>> from scipy import linalg as la

# Get 2000 random points in the 2-D domain [-1,1]x[-1,1].
>>> points = np.random.uniform(-1, 1, (2,2000))

# Determine how many points are within the circle.
>>> lengths = la.norm(points, axis=0)
>>> num_within = np.count_nonzero(lengths < 1)

# Estimate the circle's area.
>>> 4 * (num_within / 2000)
3.198
\end{lstlisting}

The estimate $\pi \approx 3.198$ isn't perfect, but it only differs from the true value of $\pi$ by about $0.0564$.
On average, increasing the number of sample points decreases the estimate error.

\begin{figure}[H] % Estimating pi.
\captionsetup[subfigure]{justification=centering}
\centering
\begin{subfigure}{.32\textwidth}
    \centering
    \includegraphics[width=\textwidth]{figures/mc_circle_0500.pdf}
\end{subfigure}
%
\begin{subfigure}{.32\textwidth}
    \centering
    \includegraphics[width=\textwidth]{figures/mc_circle_2000.pdf}
\end{subfigure}
%
\begin{subfigure}{.32\textwidth}
    \centering
    \includegraphics[width=\textwidth]{figures/mc_circle_8000.pdf}
\end{subfigure}
\caption{Estimating the area of the unit circle using random points.}
\end{figure}

\begin{problem}
The $n$-dimensional \emph{open unit ball} is the set $U_n = \{\x\in\mathbb{R}^n\mid \|\x\|_2 < 1\}$.
Write a function that accepts an integer $n$ and a keyword argument $N$ defaulting to $10^4$.
Estimate the volume of $U_n$ by drawing $N$ points over the $n$-dimensional domain $[-1,1]\times[-1,1]\times\cdots\times[-1,1]$.
\\(Hint: the volume of $[-1,1]\times[-1,1]\times\cdots\times[-1,1]$ is $2^n$.)

When $n=2$, this is the same experiment outlined above so your function should return an approximation of $\pi$.
The volume of the $U_3$ is $\frac{4}{3}\pi \approx 4.18879$, and the volume of $U_4$ is $\frac{\pi^2}{2} \approx 4.9348$.
Try increasing the number of sample points $N$ to see if your estimates improve.
\label{prob:unit-ball-volume}
\end{problem}

\section*{Integral Estimation} % ==============================================

The strategy for estimating $\pi$ can be formulated as an integral problem.
Define $f:\mathbb{R}^2\rightarrow\mathbb{R}$ by
\[
f(\x) =
\begin{cases}
1 &\mbox{ if $\|\x\|_2 < 1$ ($\x$ is within the unit circle)} \\
0 &\mbox{ otherwise,}
\end{cases}
\]
and let $\Omega = [-1,1] \times [-1,1]$ as before.
Then
\[
\int_{-1}^1\int_{-1}^1 f(x,y)\:dx\:dy = \int_\Omega f(\x)\:dV = \pi.
\]
To estimate the integral we chose $N$ random points $\{\x_i\}_{i=1}^N$ in $\Omega$.
Since $f$ indicates whether or not a point lies within the unit circle, the total number of random points that lie in the circle is the sum of the $f(\x_i)$.
Then the average of these values, multiplied by the volume $V(\Omega)$, is the desired estimate:
\begin{equation}
\int_\Omega f(\x)\:dV \approx V(\Omega) \frac{1}{N} \sum_{i=1}^N f(\x_i).
\label{eq:monte-carlo-integration}
\end{equation}
This remarkably simple equation can be used to estimate the integral of any integrable function $f:\mathbb{R}^n\rightarrow \mathbb{R}$ over any domain $\Omega \subset \mathbb{R}^n$ and is called the general formula for \emph{Monte Carlo integration}.

The intuition behind (\ref{eq:monte-carlo-integration}) is that $\frac{1}{N} \sum_{i=1}^N f(\x_i)$ approximates the average value of $f$ on $\Omega$, and multiplying the approximate average value by the volume of $\Omega$ yields the approximate integral of $f$ over $\Omega$.
This is a little easier to see in one dimension: for a single-variable function $f:\mathbb{R}\rightarrow\mathbb{R}$, the Average Value Theorem states that the average value of $f$ over an interval $[a,b]$ is given by
\[
f_{avg} = \frac{1}{b-a}\int_a^b f(x)\:dx.
\]
Then using the approximation $f_{avg} \approx \frac{1}{N} \sum_{i=1}^N f(x_i)$, the previous equation becomes
\begin{equation}
\int_a^b f(x)\:dx = (b - a) f_{avg} \approx V(\Omega)\frac{1}{N}\sum_{i=1}^N f(x_i),
\label{eq:monte-carlo-integration-1d}
\end{equation}
which is (\ref{eq:monte-carlo-integration}) in one dimension.
In this setting $\Omega = [a,b]$ and hence $V(\Omega) = b - a$.

\begin{problem}
Write a function that accepts a function $f:\mathbb{R}\rightarrow\mathbb{R}$, bounds of integration $a$ and $b$, and an integer $N$ defaulting to $10^4$.
Use \li{np.random.uniform()} to sample $N$ points over the interval $[a,b]$, then use (\ref{eq:monte-carlo-integration-1d}) to estimate the integral
\[
\int_a^b f(x)\:dx.
\]
Test your function on the following integrals, or on other integrals that you can check by hand.
\[
\begin{array}{c}
    \begin{array}{ccccc}
    \displaystyle\int_{-4}^2 x^2\:dx = 24
    & &
    \displaystyle\int_{-2\pi}^{2\pi} \sin(x)\:dx = 0
    & &
    \displaystyle\int_1^{10} \frac{1}{x}dx = \log(10) \approx 2.30259
    \end{array}
\\ \\
\displaystyle\int_1^5 \left|\sin(10x)\cos(10x) + \sqrt{x}\sin(3x) \right|\:dx \approx 4.502
\end{array}
\]
\label{prob:monte-carlo-integration-1d}
\end{problem}

\begin{warn}
Be careful not to use Monte Carlo integration to estimate integrals that do not converge.
For example, since $1/x$ approaches $\infty$ as $x$ approaches $0$ from the right, the integral \[\int_0^1 \frac{1}{x}\:dx\] does not converge.
Even so, attempts at Monte Carlo integration still return a finite value.
Use various numbers of sample points to see whether or not the integral estimate is converging.
\begin{lstlisting}
>>> for N in [5000, 7500, 10000]:
...     print(np.mean(1. / np.random.uniform(0, 1, N)), end='\t')
...
11.8451683722    25.5814419888    7.64364735049     # No convergence.
\end{lstlisting}
\end{warn}

\subsection*{Integration in Higher Dimensions} % ------------------------------

The implementation of (\ref{eq:monte-carlo-integration}) for a function $f:\mathbb{R}^n\rightarrow\mathbb{R}$ with $n > 1$ introduces a few tricky details, but the overall procedure is the same for the case when $n=1$.
We consider only the case where $\Omega \subset \mathbb{R}^n$ is an $n$-dimensional box $[a_1,b_1]\times[a_2,b_2]\times\cdots\times[a_n,b_n]$.
\begin{enumerate}
\item
If $n=1$ then $\Omega$ is a line, so $V(\Omega) = b_1 - a_1$.
If $n=2$ then $\Omega$ is a rectangle, and hence $V(\Omega) = (b_1 - a_1)(b_2-a_2)$, the product of the side lengths.
The volume of a higher-dimensional box $\Omega$ is also the product of the side lengths,
\begin{equation}
V(\Omega) = \prod_{i=1}^n (b_i - a_i)
\label{eq:mc-box-volume}
\end{equation}

\item It is easy to sample uniformly over an interval $[a,b]$ with \li{np.random.uniform()}, or even over the $n$-dimensional cube $[a,b]\times[a,b]\times\cdots\times[a,b]$ (such as in Problem \ref{prob:unit-ball-volume}).
However, if $a_i \ne a_j$ or $b_i \ne b_j$ for any $i\ne j$, the samples need to be constructed in a slightly different way.

The interval $[0,1]$ can be transformed to the interval $[a,b]$ by scaling it so that it is the same length as $[a,b]$, then shifting it to the appropriate location.
% \[((b - a) \cdot [0, 1]) + a = [0, b-a] + a = [a, b]\]
\[[0,1]\xrightarrow{\text{scale by } b-a} [0,b-a] \xrightarrow{\text{shift by } a} [a,b]\]
This suggests a strategy for sampling over $[a_1,b_1]\times[a_2,b_2]\times\cdots\times[a_n,b_n]$: sample uniformly from the $n$-dimensional box $[0, 1]\times[0, 1]\times\cdots\times[0, 1]$, multiply the $i$th component of each sample by $b_i - a_i$, then add $a_i$ to that component.
\begin{equation}
[0, 1]\times\cdots\times[0, 1]
\xrightarrow{\text{scale}}
[0, b_1-a_1]\times\cdots\times[0, b_n-a_n]
\xrightarrow{\text{shift}}
[a_1,b_1]\times\cdots\times[a_n,b_n]
\label{eq:mc-domain-transform}
\end{equation}
\end{enumerate}

\begin{problem}
Write a function that accepts a function $f:\mathbb{R}^n\rightarrow\mathbb{R}$, a list of lower bounds $[a_1, a_2, \ldots, a_n]$, a list of upper bounds $[b_1, b_2, \ldots, b_n]$, and an integer $N$ defaulting to $10^4$.
Use (\ref{eq:monte-carlo-integration}), (\ref{eq:mc-box-volume}), and (\ref{eq:mc-domain-transform}) with $N$ sample points to estimate the integral
\[
\int_\Omega f(\x)\:dV,
\]
where $\Omega = [a_1,b_1]\times[a_2,b_2]\times\cdots\times[a_n,b_n]$.
\\(Hint: use a list comprehension to calculate all of the $f(\x_i)$ quickly.)

Test your function on the following integrals.
\[
\begin{array}{c}
    \begin{array}{ccc}
    \displaystyle\int_0^1\int_0^1 x^2 + y^2\:dx\:dy = \frac{2}{3}
    & &
    \displaystyle\int_{-2}^1\int_1^3 3x - 4y + y^2\:dx\:dy = 54
    \end{array}
\\ \\
\displaystyle\int_{-4}^4\int_{-3}^3\int_{-2}^2\int_{-1}^1 x + y - w z^2\:dx\:dy\:dz\:dw = 0
\end{array}
\]
Note carefully how the order of integration defines the domain; in the last example, the $x$-$y$-$z$-$w$ domain is $[-1,1]\times[-2,2]\times[-3,3]\times[-4,4]$, so the lower and upper bounds passed to your function should be $[-1, -2, -3, -4]$ and $[1, 2, 3, 4]$, respectively.
% (this integral requires a very high $N$ for a good estimate)
\label{prob:montecarlo-integration-nd}
\end{problem}

\section*{Convergence} % ======================================================

Monte Carlo integration has some obvious pros and cons.
On the one hand, it is difficult to get highly precise estimates.
In fact, the error of the Monte Carlo method is proportional to $1/\sqrt{N}$, where $N$ is the number of points used in the estimation.
This means that dividing the error by $10$ requires using $100$ times more sample points.

On the other hand, the convergence rate is independent of the number of dimensions of the problem.
That is, the error converges at the same rate whether integrating a $2$-dimensional function or a $20$-dimensional function.
This gives Monte Carlo integration a huge advantage over other methods, and makes it especially useful for estimating integrals in high dimensions where other methods become computationally infeasible.

\begin{problem}
The probability density function of the joint distribution of $n$ independent normal random variables, each with mean $0$ and variance $1$, is the function $f:\mathbb{R}^n\rightarrow\mathbb{R}$ defined by
\[
f(\x) = \frac{1}{(2 \pi)^{n/2}} e^{- \frac{\x\trp\x}{2}}.
\]
% The integral of $f$ over a domain $\Omega\subset\mathbb{R}^n$ is the probability that a draw from the distribution will be in $\Omega$.
Though this is a critical distribution in statistics, $f$ does not have a symbolic antiderivative.

Integrate $f$ several times to study the convergence properties of Monte Carlo integration.

\begin{enumerate}
\item Let $n=4$ and $\Omega=[-\frac{3}{2}, \frac{3}{4}]\times[0,1]\times[0, \frac{1}{2}]\times[0,1] \subset \mathbb{R}^4$.
Define $f$ and $\Omega$ so that you can integrate $f$ over $\Omega$ using your function from Problem \ref{prob:montecarlo-integration-nd}.

\item Use \li{scipy.stats.mvn.mvnun()} to get the ``exact'' value of $F = \int_\Omega f(\x)\:dV$.
As an example, the following code computes the integral over $[-1,1]\times [-1,3]\times[-2,1] \subset \mathbb{R}^3$.
\begin{lstlisting}
>>> from scipy import stats

# Define the bounds of integration.
>>> mins = np.array([-1, -1, -2])
>>> maxs = np.array([ 1,  3,  1])

# The distribution has mean 0 and covariance I (the nxn identity).
>>> means, cov = np.zeros(3), np.eye(3)

# Compute the integral with SciPy.
>>> stats.mvn.mvnun(mins, maxs, means, cov)[0]
0.4694277116055261
\end{lstlisting}

\item Use \li{np.logspace()} to get $20$ \textbf{integer} values of $N$ that are roughly logarithmically spaced from $10^1$ to $10^5$.
For each value of $N$, use your function from Problem \ref{prob:montecarlo-integration-nd} to compute an estimate $\tilde{F}(N)$ of the integral with $N$ samples.
Compute the relative error $\frac{|F - \tilde{F}(N)|}{|F|}$ for each value of $N$.

\item Plot the relative error against the sample size $N$ on a log-log scale.
Also plot the line $1/\sqrt{N}$ for comparison.
Your results should be similar to Figure \ref{fig:monte-carlo-convergence}.
\end{enumerate}
\end{problem}

\begin{figure}[H] % Convergence of MC Integration.
\includegraphics[width=.7\textwidth]{figures/mc_convergence.pdf}
\caption{Monte Carlo integration converges at the same rate as $1/\sqrt{N}$ where $N$ is the number of samples used in the estimate.
However, the convergence is independent of dimension, which is why this strategy is so commonly used for high-dimensional integration.
}
\label{fig:monte-carlo-convergence}
\end{figure}

\begin{comment} % Turned out to not be very interesting.
\begin{problem}
Repeat the experiment of Problem \ref{prob:mc-point-convergence}, but fix $N=10^4$ and define $\Omega_n$ to be the $n$-dimensional cube $[-1,1]\times[-1,1]\times\cdots\times[-1,1]$.
For $n=5,10,20,\ldots,100$, redefine $f$ and $\Omega_n$ to fit the dimension, use SciPy to compute the correct value of the integral $F_n$, use your function from Problem \ref{} to estimate the integral, and compute the relative error.

Plot the dimension against the relative error.
\end{problem}
\end{comment}

\begin{comment}
\begin{problem}
Create a new function (based upon the function from Problem \ref{prob:montecarlo-integration-nd}) that uses a ``flawed'' random number generator that doesn't produce numbers between $-.95$ and $-1$.
Test your method on the function from Problem \ref{prob:mc_test}.
How bad is the error?
\label{prob:mc_flawed}
\end{problem}
\end{comment}

\begin{comment} % TODO

\newpage

\section*{Additional Material} % ==============================================

\subsection*{Volumes in High Dimensions} % ------------------------------------

It is difficult to gain intuition on how large or small an integral should be in a high-dimensional space.
Examining the size of the unit balls gives a little insight.
The volumes $V_n$ of the open unit balls $U_n$ satisfy the following recurrence relation.
\[
V_0 = 1, \qquad V_1 = 2, \qquad V_n = \frac{2\pi}{n}V_{n-2}\ \ \text{for}\ \ n > 1.
\]
The following figure plots the dimension $n$ against the exact volume $V_n$, using the recurrence relation.
This provides another way to check Problem \ref{prob:unit-ball-volume}.

\begin{figure}[H]
    \includegraphics[width=.7\textwidth]{figures/unit_volumes.pdf}
\end{figure}

% \begin{table}[H]
% \begin{tabular}{c|r}
% Dimension $n$ & Volume $V_n$ \\ \hline
% 0 & 1 \\
% 1 & 2 \\
% 2 & $\pi \approx 3.142$ \\
% 3 & $\frac{4}{3}\pi \approx 4.189$ \\
% 4 & $\frac{1}{2}\pi^2 \approx 4.935$ \\
% 5 & $\frac{8}{15}\pi^2 \approx 5.264$ \\
% 6 & $\frac{1}{6}\pi^3 \approx 5.168$ \\
% 7 & $\frac{16}{105}\pi^3 \approx 4.725$ \\
% 8 & $\frac{1}{24}\pi^4 \approx 4.059$ \\
% 9 & $\frac{32}{945}\pi^4 \approx 3.299$ \\
% 10 & $\frac{1}{120}\pi^5 \approx 2.550$ \\
% \end{tabular}
% \end{table}

Note that the volume of the unit ball increases until $n=5$, at which point the volume decreases.
This is interesting because although initially the notion of ``more dimensions'' seems to go with ``more space,'' the restriction $\|\x\|_2 < 1$ becomes more and more limiting.

\subsection*{Other Monte Carlo Methods} % -------------------------------------

The idea of using random sampling for computations is not unique to numerical integration.
For example, \emph{Markov Chain Monte Carlo} (MCMC) methods combine strategic random sampling with the theory of Markov Chains to create a way to sample from difficult distributions.
% TODO: overview of other Monte Carlo methods.
\end{comment}

max_issues_repo_name: frigusgulo/Labs

max_forks_repo_name: frigusgulo/Labs


text:
\section{Euler's Method}
The object of \emph{Euler's method} is to obtain approximations to the well-posed initial-value problem
\[
\frac{\D y}{\D t}=f(t,y),\quad a\leq t\leq b,\quad y(a)=\alpha
\]
We will use Taylor's Theorem to derive Euler's method. Suppose that $y(t)$, the unique solution has two continuous derivations on $[a,b]$, so that for each $i=0,1,\cdots,N-1$
\[
y(t_{i+1})=y(t_i)+(t_{i+1}-t_i)y'(t_i)+\frac{(t_{i+1}-t_i)^2}{2}y''(\xi_i)
\]
for some number $\xi_i$ in $t_i,t_{i+1}$. Because $h=t_{i+1}-t_i$, we have
\[
y(t_{i+1})=y(t_i)+hy'(t_i)+\frac{h)^2}{2}y''(\xi_i)
\]
Euler's method constructs $\omega_i\approx y(t_i)$, for each $i=1,2,\cdots,N$, by deleting the remainder term, then Euler's method is
\begin{align*}
  \begin{cases}
    \omega_0=\alpha & \\
    \omega_{i+1}=\omega_i+hf(t_i,\omega_i) & for i=0,1,\cdots,N-1
  \end{cases}
\end{align*}
\subsection{Errors Bounds for Euler's Method}
\begin{theo}
Suppose $f$ is continuous and satisfies a Lipschitz condition with constant $L$ on
\[
D=\left\{(t,y)|a\leq t\leq b,y\in\mathbb{R}\right\}
\]
and that a constant $M$ exists with
\[
\abs{y''(t)}\leq M,\quad \text{for all $t\in [a,b]$}
\]
where $y(t)$ denotes the unique solution to the initial-value problem
\[
\frac{\D y}{\D t}=f(t,y),\quad a\leq t\leq b,\quad y(a)=\alpha
\]
Let $\omega_0,\cdots,\omega_N$ be the approximations generated by Euler's method for some positive integer $N$, then for each $i=0,1,\cdots,N$
\[
\abs{y(t_i)-\omega_i}\leq \frac{hM}{2L}\left[e^{L(t_i-a)}-1\right].
\]
\end{theo}
\begin{proof}
\begin{align*}
  \abs{y_{i+1}-\omega_{i+1}} &\leq \abs{y_i-\omega_i}+h\abs{f(t_i,y_i)-f(t_i,\omega_i)}+\frac{h^2}{2}\abs{y''(\xi_i)} \\
  &\leq (1+hL)\abs{y_i-\omega_i}+\frac{h^2M}{2} \\
  &\leq e^{(i+1)hL}(\abs{y_0-\omega_0}+\frac{h^2M}{2hL})-\frac{h^2M}{2hL} \\
  &= \frac{hM}{2L}\left(e^{(t_{i+1}-a)L}-1\right).
\end{align*}
\end{proof}
\begin{theo}
If $u_0,u_1,\cdots,u_N$ be the approximations and $\abs{\delta_i}<\delta$, then
\[
\abs{y(t_i)-u_i}\leq\frac{1}{L}\left(\frac{hM}{2}+\frac{\delta}{h}\right)\left[e^{L(t_i-a)}-1\right]+\delta e^{L(t_i-a)}
\]
The minimal value of $E(f)$ occurs when $h=\sqrt{\frac{2\delta}{M}}$
\end{theo}

max_stars_repo_name: Iydon/NumericalAnalysisNotes

max_issues_repo_name: iydon/NumericalAnalysisNotes

max_forks_repo_name: iydon/NumericalAnalysisNotes