% vim: tw=50
% 02/02/2023 11AM

\subsection{Maximum likelihood Estimation}

Let $X = (X_1, \ldots, X_n)$ have f=joint pdf (or
pmf) $f_X(x \mid \theta)$.

\begin{flashcard}
\begin{definition*}[Likelihood function]
    \cloze{
    The likelihood function is
    \[ L : \theta \mapsto f_X(X \mid \theta) \]
    The maximum likelihood estimator (mle) is any
    value of $\theta$ maximising $L(\theta)$.
}
\end{definition*}
\end{flashcard}

\noindent
If $X_1, \ldots, X_n$ are IID each with pdf (or
pmf) $f_X(\bullet \mid \theta)$, then
\[ L(\theta) = \prod_{i = 1}^n f_X(x_i \mid
\theta) \]
We'll denote the logarithm
\[  l(\theta) = \log L(\theta) = \sum_{i = 1}^n
\log f_X(x_i \mid \theta) \]

\begin{example*}
    $X_1, \ldots, X_n \iidsim \Ber(\theta)$.
    \[ l(\theta) = \left(\sum X_i\right) \log\theta + \left(n -
    \sum X_i\right) \log(1 - \theta) \]
    \[ \pfrac{l}{\theta} = \frac{\sum X_i}{\theta}
    - \frac{n - \sum X_i}{1 - \theta} \]
    This is equal to 0 if and only if $\theta =
    \frac{1}{n} \sum X_i = \ol{X}$. Hence $\ol{X}$
    is the mle for $\theta$. This is unbiased as
    $\EE \ol{X} = \theta$.
\end{example*}

\begin{example*}
    $X_1, \ldots, X_n \iidsim \normaldist(\mu,
    \sigma^2)$
    \[ l(\mu, \sigma^2) = -\frac{n}{2} \log(2\pi)
    - \frac{n}{2}\log\sigma^2 -
    \frac{1}{2\sigma^2} \sum_{i = 1}^n (x_i -
    \mu)^2 \]
    This is maximised when $\pfrac{l}{\mu} =
    \pfrac{l}{\sigma^2} = 0$
    \[ \pfrac{l}{\mu} = -\frac{1}{\sigma^2}
    \sum_{i = 1}^n (X_i - \mu) \]
    equal to 0 when $\mu = \ol{X}$ ($\forall
    \sigma^2$)
    \[ \pfrac{l}{\sigma^2}=  -\frac{n}{2\sigma^2}
    + \frac{1}{2\sigma^4} \sum_{i = 1}^n (X_i -
    \mu)^2 \]
    This is equal to 0 when $\sigma^2 =
    \frac{1}{n} \sum_{i = 1}^n (X_i - \ol{X})^2 =
    \frac{1}{n} S_{XX}$.
    Hence $(\hat{\mu}, \hat{\sigma}^2) = (\ol{X},
    S_XX/n)$ are the mle in this model.

    \myskip
    Note that $\ol{\mu} = \ol{X}$ is unbiased. Is
    $\hat{\sigma}^2$ biased? We could compute $\EE
    \hat{\sigma}^2$ directly. Later in the course,
    we'll show that
    \[ \frac{S_{XX}}{\sigma^2} =
    \frac{n\hat{\sigma}^2}{\sigma^2} \sim \chi_{n
    - 1}^2 \]
    \[ \EE \hat{\sigma}^2 = \EE(\chi_{n - 1}^2)
    \frac{\sigma^2}{n} = \frac{n - 1}{n} \sigma^2
    \neq \sigma^2 \]
    So $\hat{\sigma}^2$ is biased, but
    asymptotically unbiased:
    \[ \bias(\hat{\sigma}^2) \stackrel{n \to
    \infty}{\longrightarrow} 0 \qquad \forall
    \sigma^2 \]
\end{example*}

\begin{example*}
    $X_1, \ldots, X_n \iidsim \Unif[0, \theta]$
    \[ L(\theta) = \frac{1}{\theta^n}
    \mathbbm{1}_{\{\max_i X_i \le \theta\}} \]
    \begin{center}
        \includegraphics[width=0.6\linewidth]
        {images/8f36d016a2ee11ed.png}
    \end{center}
    We an see from the plot that $\hat{\theta} =
    \max_i X_i$ is the mle for $\theta$. Last time we
    started from unbiased estimator $\tilde{\theta} =
    2X_1$ and using the R-B theorem we found an
    estimator
    \[ \hat{\theta} = \frac{n + 1}{n} \max_i X_i \]
    This is also unbiased. So in this model the mle is
    biased as
    \[ \EE\hat{\theta}_{mle} = \EE \left[ \frac{n +
    1}{n} \hat{\theta} \right] = \frac{n}{n + 1}
    \theta \]
    \emph{but} it is asymptotically unbiased.
\end{example*}

\subsubsection*{Properties of the mle}

\begin{enumerate}[(1)]
    \item If $T$ is a sufficient statistic then
        the mle is a function of $T(X)$. By the
        factorisation criterion:
        \[ L(\theta) = g(T(x), \theta) h(x) \]
        If $T(x) = T(y)$ the likelihood function
        with data $x$ or $y$ is the same up to a
        multiplicative constant. Hence, the mle in
        each case is the same.
    \item If $\phi = h(\theta)$ where $h$ is a
        bijection, then the mle of $\phi$ is
        $\hat{\phi} = h(\hat{\theta})$ where
        $\hat{\theta}$ is the mle of $\theta$.
    \item Asymptotic normality: $\sqrt{n}
        (\hat{\theta} - \theta)$ is approximately
        normal with mean $0$ when $n$ is large.
        Under some regularity conditions, for a
        ``nice set'' $A$,
        \[ \PP(\sqrt{n}(\hat{\theta} - \theta) \in
        A) \stackrel{n \to
        \infty}{\longrightarrow} \PP(z \in A) \]
        where $z \sim \normaldist(0,
        \ul{\Sigma})$. This holds for all
        ``regular'' values of $\theta$.

        \myskip
        Here $\Sigma$ is some function of $l$, and
        there is a theorem (Cramer-Rao) which says
        this is the smallest variable attainable.
    \item Sometimes if the mle is not available
        analytically, we can find it numerically.
\end{enumerate}

\subsubsection*{Confidence Intervals}

\begin{example*}
    Vaccine has $76\%$ efficacy in a 3-month
    period, with a $95\%$ confidence interval
    $(59\%, 86\%)$
\end{example*}

\begin{flashcard}[confidence-interval-defn]
\begin{definition*}[Confidence Interval]
    \cloze{
    A $(100 \cdot \gamma)\%$-confidence interval
    for a parameter $\theta$ is a random interval
    $(A(X), B(X))$ such that
    \[ \PP(A(X) \le \theta \le B(X)) = \gamma \]
    for all values of $\theta$. ($A$ and $B$ are
    random, and $\theta$ is fixed).
}
\end{definition*}
\end{flashcard}

\noindent
Correct or frequentist interpretation:

\myskip
There exists some fixed true parameter $\theta$.
We repeat the experiment many times. On average,
$100 \cdot \gamma\%$ of the time the interval
$(A(X), B(X))$ contains $\theta$.

\myskip
Misleading interpretation: \\
``Having observed $X = x$, there is a probability
$\gamma$ that $\theta$ is in $(A(x), B(x))$.''

\begin{example*}
    $X_1, \ldots, X_n \iidsim \normaldist(\theta,
    1)$. Find a $95\%$ confidence interval for
    $\theta$. We know that
    \begin{align*}
        \ol{X}
        =& \frac{1}{n} \sum X_i \sim
        \normaldist \left( \theta, \frac{1}{n}
        \right) \\
        \implies z
        \defeq& \sqrt{n} (\ol{X} - \theta) \sim
        \normaldist(0, 1)
    \end{align*}
    $z$ has this distribution for all $\theta$.

    \myskip
    Let $z_1, z_2$ be any two numbers such that
    $\Phi(z_2) - \Phi(z_1) = 0.95$.
    \begin{center}
        \includegraphics[width=0.6\linewidth]
        {images/87cc9b14a2f111ed.png}
    \end{center}
    Then
    \[ \PP(z_1 \le \sqrt{n}(\ol{X} - \theta) \le
    z_2) = 0.95 \]
    Rearrange:
    \[ \PP\left(\ol{X} - \frac{z_2}{\sqrt{n}} \le
    \theta \le \ol{X} \frac{z_1}{\sqrt{n}}\right)
    = 0.95 \]
    Then $\left( \ol{X} - \frac{z_2}{\sqrt{n}},
    \ol{X} + \frac{z_2}{\sqrt{n}} \right)$ is a
    $95\%$ confidence interval. How to choose
    $z_1, z_2$? Usually we minimise the width of
    interval. In this case this is achieved by
    \[ z_1 = \Phi^{-1}(0.025), \qquad z_2 =
    \Phi^{-1}(0.975) \]
\end{example*}
