% vim: tw=50
% 08/11/2022 10AM

\noindent
Recall that
\[ \tau_k = \inf\{n \ge 1 : X_n = k\} \]
$k$ is recurrent if and only if $\PP_k(\tau_k <
\infty) = 1$. $k$ is positive recurrent if $\EE_k[
\tau_k] < \infty$. Otherwise $k$ is
null-recurrent.

\begin{flashcard}[positive-recurrent-states-equivalences]
\begin{theorem*}
    Let $P$ be an irreducible matrix. Then the
    following are equivalent:
    \begin{enumerate}[(1)]
        \item All states are positive
            recurrent
        \item \cloze{Some state is positive
            recurrent}
        \item \cloze{There exists an invariant
            distribution $\pi$.}
    \end{enumerate}
    If any of the above holds, then
    \[ \cloze{\pi_k = \frac{1}{\EE_k[\tau_k]}} \]
\end{theorem*}
\end{flashcard}

\begin{proof}
    \phantom{}
    \begin{enumerate}
        \item[(1) $\implies$ (2)] Obvious.
        \item[(2) $\implies$ (3)] Let $k$ be the
            positive recurrent state.
            \[ \forall i \quad \nu_k(i) = \EE_k
            \left[ \sum_{l = 0}^{\tau_k - 1}
            \mathbbm{1}(X_l = i) \right] \]
            implies $k$ is also recurrent so by
            theorem from last time, $\nu_k P = \nu_k$.
            $\nu_k$: invariant measure.
            \[ \sum_{i \in I} \nu_k(i) = \EE_k \left[
            \sum_{l = 0}^{\tau_k - 1} \sum_{i \in I}
            \mathbbm{1}(X_l = i) \right] =
            \EE_k[\tau_k] \]
            Since $k$ is positive recurrent, implies
            $\EE_k[\tau_k] < \infty$. So we can define
            \[ \pi_i = \frac{\nu_k(i)}{\EE_k[\tau_k]} \]
            invariant distribution.
        \item[(3) $\implies$ (1)] Let $\pi$ be the
            invariant distribution. Let $k$ be a
            state. Need to show $k$ is positive
            recurrent. First show $\pi_k > 0$.
            Exists $i \in I$ such that $\pi_i >
            0$. $\pi = \pi P = \pi P^n$ for all
            $n$.
            \[ \pi_k = \sum_k \pi_j P^n(j, k) \]
            Take $n$ such that $P^n(i, k) > 0$
            (irreducibility of $P$). Then
            \[ \pi_k \ge \pi_i P^n(i, k) > 0 \]
            Define $\lambda_i =
            \frac{\pi_i}{\pi_k}$: invariant
            measure, with $\lambda_k = 1$. So
            since $P$ is irreducible $\lambda \ge
            \nu_k$, i.e. $\forall i$ $\nu_k(i) \le
            \lambda_i$.
            \[ \EE_K[\tau_k] = \sum_{i \in I} \nu_k(i)
            \le \sum_{i \in I} \lambda_i =
            \frac{1}{\pi_k} \]
            So $\EE_k[\tau_k] \le \frac{1}{\pi_k}
            < \infty$. So $k$ is positive
            recurrent.
    \end{enumerate}
    Suppose (1), (2), (3) hold. Let $k$ be a
    state. Then $k$ is positive recurrent. Define
    $\lambda_i = \frac{\pi_i}{\pi_k}$: invariant
    measure with $\lambda_k = 1$. Since $P$ is
    recurrent, $\lambda = \nu_k$ (that is,
    $\lambda_i = \nu_k(i)$ for all $i$). So
    \[ \sum_{i \in I} \lambda_i = \sum_{i \in I}
    \nu_k(i) \]
    so
    \[ \frac{1}{\pi_k} = \EE_k[\tau_k] \]
\end{proof}

\begin{flashcard}[expected-hitting-number-in-terms-of-inv-dist]
\begin{corollary*}
    $P$ irreducible, $\pi$ invariant distribution.
    Then for all $x, y$, $\nu_x(y) =
    \cloze{\frac{\pi(y)}{\pi(x)}}$.
\end{corollary*}
\end{flashcard}

\begin{example*}
    Simple random walk on $\ZZ$. $P(x, x + 1) =
    P(x, x - 1) = \half$. $P$ is recurrent ($d =
    1$). Does there exist an invariant
    distribution? $\pi = \pi P$. Need
    \[ \pi_i = \half \pi_{i - 1} + \half \pi_{i +
    1} \]
    Then $\pi = 1$ for all $i$ satisfies $\pi =
    \pi P$. Since $P$ is recurrent, all invariant
    measures have to be multiples of $\pi_i = 1$
    for all $i$. So there does not exist invariant
    distribution, so not positive recurrent.
\end{example*}

\begin{example*}
    $\ZZ$, $P(x, x + 1) = p$, $P(x, x - 1) = q$, $p
    + q = 1$, $p > q$. Need $\pi = \pi P$, need
    \[ \pi_i = p \pi_{i - 1} + q\pi_{i + 1} \]
    Solve to get
    \[ \pi = a + b \left( \frac{p}{q} \right)^i \]
    is an invariant measure for any choice of $a,
    b$. So no uniqueness up to multiplicative
    factors. Indeed, $P$ is transient.
\end{example*}

\begin{example*}
    Simple random walk on $\ZZ^3$ transient.
    $\pi_i = 1$ for all $i \in \ZZ^3$ invariant
    measure. This shows that the existence of an
    invariant \emph{measure} does not imply
    recurrence.
\end{example*}

\begin{example*}
    $\ZZ_+$, $P(x, x + 1) = p$, $P(x, x - 1) = q$,
    $p + q = 1$, $p < q$. $P(0, 1) = p$, $P(0, 0)
    = q$. Look for $\pi$ such that $\pi = \pi P$.
    \[ \pi_i = p \pi_{i - 1} + q \pi_{i + 1} \quad
    i \ge 1 \]
    \[ \pi_0 = q \pi_1 + \pi_0 q \]
    \[ \pi_1 = \pi_0 \frac{p}{q}, \quad \pi_i =
    \left( \frac{p}{q} \right)^i \pi_0 \quad
    \forall i \ge 1 \]
    $p < q$, set $\pi_0 = 1 - \frac{p}{q}$, to get
    \[ \pi_i = \left( \frac{p}{q} \right)^i \left(
    1 - \frac{p}{q} \right), \quad i \ge 0 \]
    so there exists an invariant distribution, so
    positive recurrent.
\end{example*}

\subsubsection*{Time Reversibility}

\begin{flashcard}[time-reversing-matrix]
\begin{proposition*}
    $P$ irreducible, $\pi$ invariant distribution.
    Fix $N \in \NN$ and $X_0 \sim \pi$. Define
    $Y_n = X_{N - n}$, $0 \le n \le N$. Then
    $(Y_n)_{0 \le n \le N}$ is a Markov chain with
    transition matrix
    \[ \hat{P}(x, y) = \cloze{\frac{\pi(y)}{\pi(x)} P(y,
    x)} \]
    and $\pi$ is an invariant distribution. If $P$
    is irreducible, then $\hat{P}$ is too.
\end{proposition*}
\end{flashcard}

\begin{proof}
    $\hat{P}$ is a transition matrix, since
    \[ \sum_y \hat{P}(x, y) = \sum_y
    \frac{\pi(y)}{\pi(x)} P(y, x) =
    \frac{\pi(x)}{\pi(x)} = 1 \]
    Let $y_0, \dots, y_N \in I$. Then
    \begin{align*}
        \PP(Y_0 = y_0, \dots, Y_N = y_N)
        &= \PP(X_n = y_0, \dots, X_0 = y_N) \\
        &= \PP(X_0 = y_N, \dots, X_n = y_0) \\
        &= \pi(y_N) P(y_N, y_{N - 1}) \cdot P(y_1,
        y_0) \\
        &= \pi(y_{N - 1}) \hat{P}(y_{N - 1}, y_N)
        P(y_{N - 1}, y_{N - 2}) \cdots P(y_1, y_0)
        \\
        &= \cdots \\
        &= \pi(y_0) \hat{P}(y_0, y_1) \cdots
        \hat{P}(y_{N - 1}, y_N)
    \end{align*}
    so $Y$ is $\Markov(\pi, \hat{P})$. Check
    $\hat{P}$ has invariant distribution $\pi$.
    Need to show $\pi \hat{P} = \pi$.
    \[ \sum_x \pi(x) \hat{P}(x, y) = \sum_x \pi(x)
    \frac{\pi(y)}{\pi(x)} P(y, x) = \pi(y) \]
    so $\pi \hat{P} = \pi$.
\end{proof}

\myskip
Also, $\hat{P}$ is irreducible.
