% vim: tw=50
% 08/03/2022 11AM

\setcounter{customexample}{1}
\begin{example*}
    $X \sim \mathrm{N}(\mu, \sigma^2)$
    \[ f_X(x) = \frac{1}{\sqrt{2\pi\sigma^2}} \exp
    \left( -\frac{(x - \mu)^2}{2\sigma^2} \right) \]
    \[ m_X(\theta) = \exp \left( \theta \mu +
    \frac{\theta^2 \mu^2}{2} \right) \]
    So $X_1 \sim \mathrm{N}(\mu_1, \sigma_1^2)$,
    $X_2 \sim \mathrm{N}(\mu_2, \sigma_2^2)$
    independent.
    \begin{align*}
        m_{X_1 + X_2}(\theta)
        &= \exp \left( \theta \mu_1 +
        \frac{\theta^2 \mu_1^2}{2} \right) \exp
        \left( \theta \mu_2 + \frac{\theta^2
        \sigma^2}{2} \right) \\
        &= \ub{\exp \left( \theta (\mu_1 + \mu_2) +
        \frac{\theta^2}{2} (\sigma_1^2 +
        \sigma_2^2) \right)}_{\text{MGF of
        $\mathrm{N}(\mu_1 + \mu_2, \sigma_1^2 +
        \sigma_2^2)$}}
    \end{align*}
\end{example*}

\subsubsection*{Convergence of Random Variables}

\begin{definition*}
    Let $(X_n)_{n \ge 1}$ and $X$ be random
    variables. We say $X_n$ \emph{converges to $X$
    in distribution} and write $X_n
    \stackrel{d}{\to} X$ if
    \[ F_{X_n}(x) \to F_X(x) \tag{$*$} \]
    for all $x \in \RR$ which are continuity
    points of $F_x$.
\end{definition*}

\setcounter{customexample}{0}
\begin{example}
    \[ X_n = \frac{1}{n} \mathrm{Unif}(\{1, \dots,
    n\}) \qquad X \sim \mathrm{Uni}[0, 1] \]
    \begin{center}
        \includegraphics[width=0.6\linewidth]
        {images/e77924b2a30911ec.png}
    \end{center}
    \begin{itemize}
        \item $F_x$ continuous
        \item ($*$) holds for all $x \in [0, 1]$.
    \end{itemize}
\end{example}

\begin{example}
    \[ X_n = \begin{cases}
        0 & \text{with $\PP = \half$} \\
        1 + \frac{1}{n} & \text{with $\PP = \half$}
    \end{cases} \]
    \[ X_n \to \mathrm{Bern} \left( \half \right) \]
    since $F_{X_n}(x) = \half$ for all $x \in (0,
    1)$, $F_{X_n}(x) = 1$ for all $x > 1$. When
    $n$ is large
    \[ F_{X_n}(1) = \half \qquad F_X(1) = 1 \]
    But $F_X(\bullet)$ has a discontinuity at $x =
    1$. (i.e. deterministic convergence of reals)
\end{example}

\subsubsection*{Consequences}

\begin{enumerate}[(1)]
    \item If $X$ is a constant $c$, then
        equivalent to:
        \[ \forall \eps > 0 \qquad \PP( |X_n - c|
        > \eps ) \to 0 \]
        as $n \to \infty$. ``convergence in
        probability to constant''.
    \item If $X$ is a continuous random variable:
        $X_n \stackrel{d}{\to} X$. Usage:
        \[ \PP(a \le X_n \le b) \to \PP(a \le X
        \le b) \]
        for all $a, b \in [-\infty, \infty]$.
        \begin{note*}
            Does not say that \emph{densities}
            converge. For example, in Example 1 no
            density.
        \end{note*}
\end{enumerate}

\subsubsection*{Laws of Large Numbers}

$\frac{S_n}{n} ``\to'' \mu$.

\begin{theorem*}[Weak LLN]
    Setup: $(X_n)_{n \ge 1}$ IID with $\mu =
    \EE[X_1] < \infty$. Set
    \[ S_n = X_1 + \cdots + X_n \qquad \forall n
    \ge 0 \]
    Then $\forall \eps > 0$:
    \[ \PP \left( \left| \frac{S_n}{n} - \mu
    \right| > \eps \right) \to 0 \]
    as $n \to \infty$.
\end{theorem*}

\begin{proof}
    (assume $\mathrm{Var}(X_1) = \sigma^2 <
    \infty$)
    \begin{align*}
        \PP \left( \left| \frac{S_n}{n} - \mu
        \right| > \eps \right)
        &= \PP(|S_n - n\mu| > \eps n) \\
        & \le \frac{\mathrm{Var}(S_n)}{\eps^2 n^2}
        \\
        &= \frac{n \sigma^2}{\eps^2 n^2} \\
        &\to 0
    \end{align*}
    as $n \to \infty$. (Note that $\eps$ is fixed,
    not $\eps \to 0$!)
\end{proof}

\subsubsection*{Central Limit Theorem}

\begin{theorem*}[CLT]
    Same setup as previous. Demand $\sigma^2 <
    \infty$. Then
    \[ \frac{S_n - n\mu}{\sqrt{n\sigma^2}}
    \stackrel{d}{\to} \mathrm{N}(0, 1) \]
    as $n \to \infty$.
\end{theorem*}

\noindent
Discussion: three stage summary
\begin{enumerate}[(1)]
    \item Distribution of $S_n$ concentrated on
        $n\mu$ (WLLN)
    \item Fluctuations around $n\mu$ have order
        $\sqrt{n}$ (New and important)
    \item Shape is normal (Detail)
\end{enumerate}
Usage:
\begin{enumerate}[(i)]
    \item $S_n \stackrel{d}{\approx}
        \mathrm{N}(n\mu, n\sigma^2)$
    \item \eqnoskip
        \begin{align*}
            \PP(a \le S_n \le b)
            &= \PP \left( \frac{a -
            n\mu}{\sqrt{n\sigma^2}} \le \frac{S_n
            - n\mu}{\sqrt{n\sigma^2}} \le \frac{b
            - n\mu}{\sqrt{n\sigma^2}} \right) \\
            &\approx \PP \left( \frac{a -
            n\mu}{\sqrt{n\sigma^2}} \le Z \le
            \frac{b - n\mu}{\sqrt{n\sigma^2}} \right)
        \end{align*}
        Get a nice answer if $a = n\mu +
        z_a\sqrt{n}$ and $b = n\mu + z_b\sqrt{n}$.
\end{enumerate}

\begin{theorem*}[Continuity theorem for MGFs]
    $(X_n), X$ have MGFs $m_{X_n}(\bullet),
    m_X(\bullet)$
    \begin{itemize}
        \item $m_X(\theta) < \infty$ for $\theta
            \in (-\eps, \eps)$
        \item if $m_{X_n}(\theta) \to m_X(\theta)$
            for all $\theta$ such that
            $m_X(\theta) < \infty$.
    \end{itemize}
    Then $X_n \stackrel{d}{\to} X$.
\end{theorem*}

\begin{proof}
    Part II Probability and Measure.
\end{proof}

\myskip
Idea: Expand $m_X(\theta)$ as Taylor series around
0.
\begin{align*}
    m_X(\theta)
    &= 1 + m_X'(0)\theta + \frac{m_X''(0)}{2!}
    \theta^2 + \cdots \\
    &= 1 + \theta \EE[X] + \frac{\theta^2}{2}
    \EE[X^2] + o(\theta^2)
\end{align*}
Proof: (WLLN via MGFs).

\begin{remark*}
    Know MGF of $S_n$. Want to study the MGF of
    $\frac{S_n}{n}$.
    \begin{align*}
        m_{\frac{S_n}{n}}(\theta)
        &= \EE[e^{\theta \frac{S_n}{n}}] \\
        &= \EE[e^{\frac{\theta}{n} S_n}] \\
        &= m_{S_n} \left( \frac{\theta}{n} \right)
        \\
        &= m_{X_1} \left( \frac{\theta}{n} \right)
        \cdots m_{X_n} \left( \frac{\theta}{n}
        \right) \\
        &= \left( 1 + \mu \frac{\theta}{n} +
        o(\theta) \right)^n \\
        &\to e^{\mu\theta}
    \end{align*}
    MGF of the random variable $X = \mu$ with $\PP
    = 1$. So $\frac{S_n}{n} \stackrel{d}{\to} \mu$
    by the continuity theorem.
\end{remark*}

\begin{theorem*}[Strong LLN]
    Same setup: Then
    \[ \PP \left( \frac{S_n}{n} \to \mu \text{ as
    $n \to \infty$} \right) = 1 .\]
    ``almost sure convergence'' or ``convergence
    with probability 1''.
\end{theorem*}
