% vim: tw=50
% 10/03/2022 11AM

\begin{proof}[(CLT with MGFs)]
    Assume WLOG $\mu = 0$ and $\sigma^2 = 1$. (So
    $\EE[X_i^2] = 1$). (In general $X \mapsto
    \frac{X - \mu}{\sqrt{\sigma^2}}$). \\
    \ul{Goal}:
    \[ \frac{S_n}{\sqrt{n}} \stackrel{d}{\to}
    \mathrm{N}(0, 1) \]
    Study MGF of $\frac{S_n}{\sqrt{n}}$.
    \begin{align*}
        m_{X_i}(\theta)
        &= 1 + \frac{\theta^2}{2} + o \left(
        \frac{1}{n} \right) \\
        m_{\frac{S_n}{\sqrt{n}}}(\theta)
        &= \EE[e^{\theta \frac{S_n}{\sqrt{n}}}] \\
        &= \EE[e^{\frac{\theta}{\sqrt{n}}S_n}] \\
        &= m_{S_n} \left( \frac{\theta}{\sqrt{n}}
        \right) \\
        &= \left( m_{X_1} \left( \frac{\theta}{\sqrt{n}}
        \right) \right)^n \\
        &= \left( 1 + \frac{\theta^2}{2n} +
        o \left( \frac{1}{n} \right) \right)^n \\
        &\to e^{\frac{\theta^2}{2}}
    \end{align*}
\end{proof}

\subsubsection*{Inequalities for $\EE[f(X)]$}

Motivation: $f(x) = x^2$. We know
\[ \EE[f(X)] \ge f(\EE[X]) \]
via $\mathrm{Var}(X) \ge 0$. What about general
$f$?

\begin{definition*}
    A function $f : \RR \to \RR$ is \emph{convex}
    if $\forall\,\,x, y \in \RR$ and $t \in [0,
    1]$,
    \[ f(tx + (1 - t)y) \le tf(x) + (1 - t)f(y) \]
    \begin{center}
        \includegraphics[width=0.6\linewidth]
        {images/d8b19e44a31f11ec.png}
    \end{center}
    (Aside: region above $f$ is \emph{convex} in
    $\RR^2$.)
\end{definition*}

\noindent
\ul{Consequence}: $\forall\,\,y$ there exists a
line $l(x) = mx + c$ such that
\begin{itemize}
    \item $l(x) \le f(x)$ for all $x$
    \item $l(y) = f(y)$
\end{itemize}

\begin{proof}
    Convexity implies that for all $x < y < z$,
    \[ \frac{f(y) - f(x)}{y - x} \le \frac{f(z) -
    f(y)}{z - y} \]
    \begin{center}
        \includegraphics[width=0.6\linewidth]
        {images/425d1828a32011ec.png}
    \end{center}
    hence
    \[ M^- := \sup_{x < y} \frac{f(y) - f(x)}{y -
    x} \le \inf_{z > y} \frac{f(z) - f(y)}{z - y}
    =: M^+ \]
    any value $m \in [M^-, M^+]$ works as the
    gradient of $l(\bullet)$.
\end{proof}

\begin{definition*}
    $f$ is concave if and only if $-f$ is convex.
    \begin{center}
        \includegraphics[width=0.6\linewidth]
        {images/67a5fea6a32011ec.png}
    \end{center}
\end{definition*}

\noindent
\ul{Fact}: if $f$ is twice differentiable then
\[ f \text{ convex} \iff f''(x) \ge 0
\,\,\forall\,\,x \]
for example $f(x) = \frac{1}{x}$ is convex on $(0,
\infty)$ and concave on $(-\infty, 0)$.

\subsubsection*{Jensen's Inequality}

\begin{theorem*}[Jensen's Inequality]
    $X$ a random variable, $f$ convex: \\
    Then $\EE[f(X)] \ge f(\EE[X])$. (reverse if
    $f$ concave)
\end{theorem*}

\begin{proof}
    Set $y = \EE[X]$ as in ($*$), $l(x) = mx + c$,
    such that $l(y) = f(y) = f(\EE[X])$ and $f \ge
    l$.
    \begin{align*}
        \EE[f(X)]
        &\ge \EE[l(X)] \\
        &= \EE[mX + c] \\
        &= m\EE[X] + c \\
        &= my + c \\
        &= f(\EE[X])
    \end{align*}
    If $f$ \emph{strictly convex}, then $\forall t
    \in (0, 1)$, $\forall x \neq y$,
    \[ f(tx + (1 - t)y) < tf(x) + (1 - t)f(y) \]
    Then equality in Jensen's inequality only if
    $X = \EE[X]$ with $\PP = 1$ (for example
    constant random variable).
\end{proof}

\myskip
Informal comment:
\[ \text{Jensen's Inequality} \ge \text{Most other
inequalities!} \]

\subsubsection*{Application to Sequences}

AM-GM inequality: $x_1, \dots, x_n \in (0,
\infty)$
\[ \frac{x_1 + \cdots + x_n}{n} \ge \left(
\prod_{i = 1}^n x_i \right)^{1/n} \]
Case $n = 2$:
\[ \frac{x + y}{2} \ge \sqrt{xy} \]
\begin{proof}
    Rearrange to get $(x - y)^2 \ge 0$.
\end{proof}

\myskip
General proof: \\
Let $X$ be a random variable taking values $\{x_1,
\dots, x_n\}$ each with probability $\frac{1}{n}$.
\\
Take: $f(x) = -\log x$. Check convex: second
derivative $\ge 0$. \\
Jensen:
\[ \EE[f(X)] \ge f(\EE[X]) \]
\[ -\frac{\log x_1 + \cdots + \log x_n}{n} \ge
-\log \left( \frac{x_1 + \cdots + x_n}{n} \right) \]
\[ \log ((x_1 \cdots x_n)^{1/n}) \le \log \left(
\frac{x_1 + \cdots + x_n}{n} \right) \]
$\log x$ and $e^x$ are increasing so
\[ \left( \prod_i x_i \right)^{1/n} \le \frac{x_1
+ \cdots + x_n}{n} \]

\subsubsection*{Sampling a Continuous Random Variable}

\begin{theorem*}
    $X$ a continuous random variable with CDF $F$.
    Then if $U \sim U[0, 1]$, we have
    \[ Y = F^{-1}(U) \sim X \]
\end{theorem*}

\begin{proof}
    Goal: find CDF of $Y$.
    \begin{align*}
        \PP(Y \le x)
        &= \PP(F^{-1}(U) \le x) \\
        &= \PP(U \le F(x)) \\
        &= F(x)
    \end{align*}
    so CDF of $Y = \text{CDF of $X$}$. So $Y \sim
    X$.
\end{proof}

\subsubsection*{Rejection Sampling}

Idea: Uniform on $[0, 1]^d$ is easy. (take
$(U^{(1)}, \dots, U^{(d)})$ IID on $U[0, 1]$.)
\begin{center}
    \includegraphics[width=0.6\linewidth]
    {images/2ce4b6dea32211ec.png}
\end{center}
What about uniform on $A$?