% vim: tw=50
% 28/01/2023 09AM

\begin{example*}
    Suppose that $X_1, \ldots, X_n \iidsim
    \normaldist(\mu, \sigma^2)$, with parameters
    $(\mu, \sigma^2)$ unknown.
    \begin{align*}
        \frac{f_X(x \mid \theta)}{f_X(y \mid
        \theta)}
        &= \frac{(2\pi \sigma^2)^{-\pi/2} \exp
        \left\{-\frac{1}{2\sigma^2} \sum_{i = 1}^n
        (x_i - \mu)^2\right\}}
        {(2\pi\sigma^2)^{-\pi/2} \exp\left\{
        -\frac{1}{2\sigma^2}\sum_{i = 1}^n (y_i -
        \mu)^2\right\}} \\
        &= \exp \left\{ -\frac{1}{2\sigma^2}
        \left( \sum_{i = 1}^n x_i^2 - \sum_{i =
        1}^n y_i^2 \right) \frac{\mu}{\sigma^2}
        (\sum_{i = 1}^n x_I - \sum_{i = 1}^n y_i \right\}
    \end{align*}
    If $\sum_{i = 1}^n x_i^2 = \sum_{i = 1}^n
    y_i^2$ and $\sum_{i = 1}^n x_i = \sum_{i =
    1}^n y_i$, this ratio does not depend on
    $(\mu, \sigma^2)$. The converse is also true:
    if the ratio does not depend on $(\mu,
    \sigma^2)$ then we must have $\sum_{i = 1}^n
    x_i^2 = \sum_{i = 1}^n y_i^2$ and $\sum_{i =
    1}^n x_i = \sum_{i = 1}^n y_i$. By the
    theorem, $T(X) = (\sum_{i = 1}^n X_i^2,
    \sum_{i = 1}^n X_i)$ is minimal sufficient.
\end{example*}

\noindent
Recall that bijections of $T$ are also minimal
sufficient. A more common way of expressing a
minimal sufficient statistic in this model is
\[ S(X) = (\ol{X}, S_{XX}) \]
\[ \ol{X} = \frac{1}{n} \sum_i X_i \qquad S_XX =
\sum_i (X_i - \ol{X})^2 \]
In this example, $(\mu, \sigma^2)$ and $T(X)$
are both 2-dimensional. In general, the parameter
and sufficient statistic can have different
dimensions.

\begin{example*}
    $X_1, \ldots, X_n \iidsim \normaldist(\mu,
    \mu^2)$, $\mu \ge 0$. Here, the minimal
    sufficient statistic is $S(X) = (\ol{X},
    S_{XX})$.
\end{example*}

\subsubsection*{Rao-Blackwell Theorem}

\begin{note*}
    So far we've written $\EE_\theta$,
    $\PP_\theta$ to denote expectations and
    probabilities in the model where $X_1, \ldots,
    X_n \iidsim f_X(\bullet \mid \theta)$. From
    now on, I'll drop the subscript $\theta$.
\end{note*}

\begin{flashcard}[rao-blackwell-thm]
\begin{theorem*}[Rao-Blackwell]
    \cloze{
    Let $T$ be a sufficient statistic for
    $\theta$. Let $\tilde{\theta}$ be some
    estimator for $\theta$, with
    $\EE(\tilde{\theta}^2) < \infty$, for all $\theta$.
    Define a new estimator $\hat{\theta} =
    \EE_\theta(\tilde{\theta} \mid T(X))$. Then,
    for all $\theta$,
    \[ \EE[(\hat{\theta} - \theta)^2] \le
    \EE[(\tilde{\theta} - \theta)^2] \]
    ($\mse(\hat{\theta}) \le
    \mse(\tilde{\theta})$). The inequality is
    strict unless $\tilde{\theta}$ is a function
    of $T(X)$.
}
\end{theorem*}
\end{flashcard}

\begin{remark*}
    $\hat{\theta}$ is a valid estimator, i.e. it
    does not depend on $\theta$, only depends on
    $X$, because $T$ is sufficient.
    \[ \hat{\theta}(T(X)) = \int
    \ub{\tilde{\theta}(X)}_{\text{estimator,
    so does not depend on $\theta$}}
    \ub{f_{X \mid T}(x \mid T)}_{\text{does not
    depend on $\theta$, because $T$ is
    sufficient}} \dd x \]
\end{remark*}

\begin{moral*}
    We can improve the $\mse$ of any estimator
    $\tilde{\theta}$ by taking a conditional
    expectation given $T(X)$.
\end{moral*}

\begin{hiddenflashcard}[rao-blackwell-proof]
Rao-Blackwell proof? \\
\cloze{
We use bias-variance decomposition. \\
By Tower property,
\[ \EE \hat{\theta} = \EE[\EE[\tilde{\theta} \mid
T]] = \EE\tilde{\theta} \]
So $\bias(\hat{\theta}) = \bias(\tilde{\theta})$
for all $\theta$. \\
By conditional variance formula,
\begin{align*}
    \Var(\tilde{\theta})
    &= \EE[\Var(\tilde{\theta} \mid T)]
    + \Var(\EE[\tilde{\theta} \mid T]) \\
    &= \EE[\Var(\tilde{\theta} \mid T)]
    + \Var(\hat{\theta})
\end{align*}
Hence $\Var(\tilde{\theta}) \ge
\Var(\hat{\theta})$, and the inequality is
strict unless $\tilde{\theta}$ is a function of
$T(X)$.
}
\end{hiddenflashcard}

\begin{proof}
    By the tower property:
    \[ \EE \hat{\theta} = \EE[\EE[\tilde{\theta}
    \mid T] = \EE\tilde{\theta} \]
    So $\bias(\hat{\theta}) =
    \bias(\tilde{\theta})$ for all $\theta$. By
    the conditional variance formula,
    \begin{align*}
        \Var(\tilde{\theta})
        &= \EE(\Var(\tilde{\theta} \mid T)) +
        \Var(\EE(\tilde{\theta} \mid T)) \\
        &= \EE[\ub{\Var(\tilde{\theta} \mid
        T)}_{\ge 0 \text{ with }\PP = 1}] +
        \Var(\hat{\theta}) \\
        \implies \Var(\tilde{\theta})
        &\ge \Var(\hat{\theta})
    \end{align*}
    for all $\theta$. Therefore
    $\mse(\tilde{\theta}) \ge \mse(\hat{\theta})$.

    \myskip
    Note:
    $\Var(\tilde{\theta} \mid T) > 0$ with
    some positive probability unless
    $\tilde{\theta}$ is a function of $T(X)$.
    So $\mse(\tilde{\theta}) >
    \mse(\hat{\theta})$ unless
    $\tilde{\theta}$ is a function of $T(X)$.
\end{proof}

\begin{example*}
    Say $X_1, \ldots, X_n \iidsim \Poisson(\lambda)$.
    We wish to estimate $\theta = \PP(X_1 = 0) =
    e^{-\lambda}$.
    \begin{align*}
        f_X(x \mid \lambda)
        &+ \frac{e^{-n\lambda} \lambda^{\sum
        x_i}}{\prod_i x_i!} \\
        \implies f_X(x \mid \theta)
        &= \frac{\theta^n(-\log\theta)^{\sum
        x_i}}{\prod_i x_i!}
    \end{align*}
    Letting $h(x) = \frac{1}{\prod x_i!}$,
    $g(T(X), \theta) =
    \theta^n(-\log\theta)^{T(X)}$, then by
    factorisation criterion, $T(X) = \sum X_i$ is
    a sufficient statistic. Let $\tilde{\theta} =
    \mathbbm{1}_{\{X_1 = 0\}}$ (unbiased: only
    uses one observation $X_1$).
    \begin{align*}
        \hat{\theta}
        &= \EE[\tilde{\theta} \mid T = t] \\
        &= \PP\left(X_1 = 0 \mid \sum_{i = 1}^n
        X_i = t\right) \\
        &= \frac{\PP\left(X_1 = 0, \sum_{i = 2}^n X_i =
        t\right)}{\PP \left( \sum_{i = 1}^n X_i =
        t\right)} \\
        &= \frac{\PP(X_1 = 0) \PP \left( \sum_{i =
        2}^n X_i = t \right)}{\PP \left( \sum_{i =
        1}^n X_i = t\right)} \\
        &= \cdots \\
        &= \left( \frac{n - 1}{n} \right)^t
    \end{align*}
    So $\hat{e} = \left( 1 - \frac{1}{n}
    \right)^{\sum x_i}$ is an estimator which by
    the Rao-Blackwell theorem has
    \[ \mse(\hat{\theta}) < \mse(\tilde{\theta}) \]
    Sanity check: What happens as $n \to \infty$?
    \[ \hat{\theta} = \left( 1 - \frac{1}{n}
    \right)^{n\ol{x}} \stackrel{n \to
    \infty}{\longrightarrow} e^{-\ol{x}} \]
    and by the Strong Law of Large Numbers,
    $\ol{X} \to \EE X_1 = \lambda$ so $\theta^n
    \approx e^{-\lambda} = \theta$ as $h$ grows
    large.
\end{example*}

\begin{example*}
    Let $X_1, \ldots, X_n \iidsim \Unif([0,
    \theta])$, $\theta$ unknown. $\theta \ge 0$.
    Recall $T(X) = \max_i X_i$ is sufficient for
    $\theta$. Let $\tilde{\theta} = 2X_1$, which
    is unbiased. Then
    \begin{align*}
        \hat{\theta}
        &= \EE[\tilde{\theta} \mid T = t] \\
        &= 2\EE[X_1 \mid \max_i X_i = t] \\
        &= 2 \EE[X_1 \mid \max_i X_i = t, \max_i
        X_i = X_1] \PP[\max_i X_i = X_1 \mid \max
        X_i = t] \\
        &~~~+ \EE[X_1 \mid \max_i X_i = t, \max_i
        X_i \neq X_1] \PP[\max_i X_i \neq X_1 \max
        X_i = t] \\
        &= \frac{2t}{n} + \frac{2(n - 1)}{n}
        \EE[X_1 \mid X_1 \le t, \max_{1 \le i \le
        n} X_i = t] \\
        &= \frac{2t}{n} + \frac{2(n - 1)}{n}
        \frac{t}{2}
    \end{align*}
    So $\hat{\theta} = \frac{n + 1}{n} \max_i X_i$ is
    a valid estimator with
    \[ \mse(\hat{\theta}) < \mse(\tilde{\theta}) \]
\end{example*}
