% vim: tw=50
% 12/02/2022 11AM

\setcounter{customexample}{0}
\begin{example}
    $X \sim \mathrm{Poisson}(\lambda)$, $\EE[X] =
    \lambda$.
    \[ \mathrm{Var}(x) = \EE[X^2] - \lambda^2 \]
    ``Falling factorial trick'': sometimes
    $\EE[X(X - 1)]$ is easier than $\EE[X^2]$.
    Here:
    \begin{align*}
        \EE[X(X - 1)]
        &= \sum_{k \ge 2} k(k - 1) e^{-\lambda}
        \frac{\lambda^k}{k!} \\
        &= \lambda^2 e^{-\lambda} \sum_{k \ge 2}
        \frac{\lambda^{k - 2}}{(k - 2)!} \\
        &= \lambda^2 \\
        \EE[X^2] = \EE[X(X - 1) + X] \\
        &= \EE[X(X - 1)] + \EE[X] \\
        &= \lambda^2 + \lambda \\
        \implies \mathrm{Var}(x)
        &= \lambda
    \end{align*}
\end{example}

\begin{example}
    $Y \sim \mathrm{Geom}(p) \in \{1, 2, 3,
    \dots\}$. $\EE[Y] = \frac{1}{p}$.
    $\mathrm{Var}(y) = \cdots = \frac{1 -
    p}{p^2}$. (left as an exercise)
\end{example}

\begin{note*}
    $\lambda$ large: $\mathrm{Var}(X) = \EE[X]$.
    \\
    $p$ small (so $Y$ large): $\mathrm{Var}(Y)
    \approx \frac{1}{p^2} = (\EE[X])^2$.
\end{note*}

\begin{example}
    $X \sim \mathrm{Bern}(p)$. $\EE[X] = 1 \times
    p = p$. $\EE[X^2] = 1^2 \times p = p$.
    \[ \mathrm{Var}(X) = p - p^2 = p(1 - p) \]
\end{example}

\begin{example}
    $X \sim \mathrm{Bin}(n, p)$, $\EE[X] = np$.
    \[ \EE[X^2] = \text{\textcolor{red}{ugly\dots}} \]
\end{example}

\noindent
Goal: Study $\mathrm{Var}(X_1 + \cdots + X_n)$ for
not independent.

\myskip
Preliminary: $\EE[\text{Products of RVs}]$.
Setting: $X$, $Y$ \emph{independent} random
variables and $f$, $f$ functions $\RR \to \RR$.
Then:
\[ \EE[f(X)g(Y)] = \EE[f(X)]\EE[g(Y)] \]
``splits as a product'' \\
\ul{Key example 1}: $f, g : f(x) = g(x) = x$. Then
$\EE[XY] = \EE[X]\EE[Y]$. \\
\ul{Key example 2}: $f(x) = g(x) = z^x$ (or
$e^{tx}$).

\begin{proof}
    \begin{align*}
        LHS
        &= \sum_{x, y \in \mathrm{Im}} f(x)g(y)
        \PP(X = x, Y = y) \\
        &= \sum_{x, y \in \mathrm{Im}} f(x)g(y)
        \PP(X = x) \PP(Y = y) \\
        &= \left[ \sum_{x \in \mathrm{Im}\,\,X} f(x)
        \PP(X = x) \right] \left[ \sum_{y \in
        \mathrm{Im}\,\,Y} g(y) \PP(Y = y) \right]
        \\
        &= \EE[f(X)] \EE[g(Y)]
    \end{align*}
\end{proof}

\subsubsection*{Sums of Independent Random Variables}

$X_1, \dots, X_n$ independent. Then
\[ \mathrm{Var}(X_1 + \cdots X_n) =
\mathrm{Var}(X_1) + \cdots \mathrm{Var}(X_n) \]

\begin{proof}
    (Suffices to prove $n = 2$ by induction). Say
    $\EE[X] = \mu$, $\EE[Y] = \nu$. Then $\EE[X +
    Y] = \mu + \nu$.
    \begin{align*}
        \mathrm{Var}(X + Y)
        &= \EE[(X + Y - \mu - \nu)^2] \\
        &= \EE[(X - \mu)^2] + \EE[(Y - \mu)^2] + 2
        \EE[(X - \mu)(Y - \nu)] \\
        &= \mathrm{Var}(X) + \mathrm{Var}(Y) +
        \EE[X - \mu]\EE[Y - \nu] \\
        \mathrm{Var}(X) + \mathrm{Var}(Y)
    \end{align*}
\end{proof}

\setcounter{customexample}{3}
\begin{example}
    $\mathrm{Var}(\mathrm{Bin}(n, p)) = np(1 -
    p)$.
\end{example}

\noindent
Goal: Study $\mathrm{Var}(X + Y)$ when $X$, $Y$
are not independent.

\begin{definition*}
    $X$, $Y$ two random variables. Their
    \emph{covariance} is
    \[ \mathrm{Cov}(X, Y) = \EE[(X - \EE[X])(Y -
    \EE[Y])] \]
    ``Measures how dependent $X$, $Y$ are, and in
    which \emph{direction}'': If $\mathrm{Cov} >
    0$ then $X$ bigger means $Y$ bigger, and
    if $\mathrm{Cov} < 0$ then $X$ bigger means
    $Y$ smaller.
\end{definition*}

\subsubsection*{Properties}

\begin{itemize}
    \item $\mathrm{Cov}(X, Y) = \mathrm{Cov}(Y,
        X)$
    \item $\mathrm{Cov}(X, X) = \mathrm{Var}(X)$.
    \item Alternative characterisation:
        \[ \mathrm{Cov}(X, Y) = \EE[XY] -
        \EE[X]\EE[Y] \]
        (often more useful, and particularly nice
        if $\EE[X] = 0$)
        \begin{proof}
            \begin{align*}
                \mathrm{Cov}(X, Y)
                &= \EE[(X - \mu)(Y - \nu)] \\
                \EE[XY] - \mu \ub{\EE[Y]}_{\nu} -
                \nu \ub{\EE[X]}_{\mu} +
                \mu \nu \\
                &= \EE[XY] - \mu \nu
            \end{align*}
        \end{proof}
    \item $c, \lambda \in \RR$:
        \begin{itemize}
            \item $\mathrm{Cov}(c, X) = 0$
            \item $\mathrm{Cov}(X + c, Y) =
                \mathrm{Cov}(X, Y)$
            \item $\mathrm{Cov}(\lambda X,
                \textcolor{red}{\lambda} Y) =
                \lambda\textcolor{red}{^2}
                \mathrm{Cov}(X, Y)$
        \end{itemize}
    \item $\mathrm{Var}(X + Y) = \mathrm{Var}(X) +
        \mathrm{Var}(Y) + 2 \mathrm{Cov}(X, Y)$
    \item Covariance is \emph{linear} in each
        argument, i.e.
        \[ \mathrm{Cov} \left( \sum \lambda_i X_i,
        Y \right) = \sum \lambda_i
        \mathrm{Cov}(X_i, Y) \]
        and (applying in two stages)
        \[ \mathrm{Cov} \left( \sum \lambda_i X_i,
        \sum \mu_j Y_j \right) = \sum_{i = 1}^n
        \sum_{j = 1}^n \lambda_i \mu_j
        \mathrm{Cov}(X_i, Y_j) \]
        ``Special case'':
        \begin{align*}
            \mathrm{Var} \left( \sum_{i = 1}^n X_i \right)
            &= \mathrm{Cov} \left( \sum_{i = 1}^n
            X_i, \sum_{i = 1}^n X_i \right) \\
            &= \sum_{i = 1}^n \mathrm{Var}(X_i)
            + \sum_{i \neq j} \mathrm{Cov}(X_i, X_j)
        \end{align*}
        (for an example, see Q11 on sheet 3)
\end{itemize}

\begin{note*}
    We have already seen that $X$, $Y$ independent
    implies $\mathrm{Cov}(X, Y) = 0$, but it is
    not the case the zero covariance implies
    independence.
\end{note*}

