% vim: tw=50
% 10/11/2022 10AM

\begin{proof}
    Let $x, y \in I$. Need to show
    $\hat{P}^n(y, x) > 0$ for some $n$. $P$ is
    irreducible, so there exists $n$ and $x_0 = x,
    \dots, x_n = y$ such that
    \[ P(x_0, x_1) \cdots P(x_{n - 1}, x_n) > 0 \]
    \begin{align*}
        \hat{P}(x_n, x_{n - 1}) \cdots
        \hat{P}(x_1, x_0)
        &= \hat{P}(x_n, x_{n - 1}) \cdots P(x_0,
        x_1) \frac{\pi(x_0)}{\pi(x_1)} \\
        &= \cdots \\
        &= P(x_0, x_1) P(x_1, x_2) \cdots P(x_{n -
        1}, x_n) \frac{\pi(x_0)}{\pi(x_n)} > 0
    \end{align*}
\end{proof}

\begin{flashcard}
\begin{definition*}
    A chain with matrix $P$ and invariant
    distribution $\pi$ is called \emph{(time)
    reversible} if
    \[ \cloze{\hat{P} = P} \]
\end{definition*}
\end{flashcard}

\noindent
That is, for all $x, y$,
\[ \hat{P}(x, y) = P(x, y) \iff \frac{\pi(y) P(y,
x)}{\pi(x)} = P(x, y) \]
$X$ is reversible if for all $x, y$,
\begin{flashcard}[detailed-balance-equation]
    \prompt{Detailed balance equation?}
    \[ \boxed{\cloze{\pi(x) P(x, y)} =
    \cloze{\pi(y)P(y, x)}} \]
\end{flashcard}
Detailed balance equation. \myskip
Equivalently, $X$ is reversible if $\forall N \in
\NN$, when $X_0 \sim \pi$, then
\[ (X_0, \dots, X_N) \sim (X_n, \dots, X_0) \]
$\PP(X_0 = x, X_1 = y) = \pi(x)P(x, y)$.

\begin{example*}
    \begin{center}
        \includegraphics[width=0.6\linewidth]
        {images/dea6710060e111ed.png}
    \end{center}
    \[ P(i, (i + 1) \bmod n) = \frac{2}{3} \quad
    P(i, (i - 1) \bmod n) = \frac{1}{3} \]
    for all $i \in \ZZ_n$. Is $P$ time reversible?
    Take $\pi_i = \frac{1}{n}$ for all $i$, then
    $P$ is not reversible, because the Detailed
    Balance equation are not satisfied, for
    example
    \[ \pi(i)P(i, i + 1) = \frac{1}{n} \frac{2}{3} \]
    but
    \[ \pi(i + 1)P(i + 1, i) = \frac{1}{n}
    \frac{1}{3} \]
\end{example*}

\begin{example*}
    \begin{center}
        \includegraphics[width=0.6\linewidth]
        {images/48ff984c60e211ed.png}
    \end{center}
    Is this time reversible? $\lambda^i = 2^i$
    invariant measure. $\pi(i) \propto 2^i$, $X_0
    \sim \pi$. Check $\pi$ satisfies Detailed
    Balance equation.
\end{example*}

\begin{lemma*}
    Let $\mu$ be a distribution satisfying
    \[ \mu(x) P(x, y) = \mu(y) P(y, x) \quad
    \forall x, y \]
    Then $\mu$ is an invariant distribution.
\end{lemma*}

\begin{proof}
    \[ \mu(y) = \sum_x \mu(x) P(x, y) = \sum_x
    \mu(y) P(y, x) = (\mu P)(y) \]
    So $\mu = \mu P$.
\end{proof}

\myskip
When looking for an invariant distribution, first
we should look for a solution to the Detailed
Balance equation. If a distribution that solves
Detailed Balance equation exists, then it is an
invariant distribution. If no solution to Detailed
Balance equation exists, then if there exists an
invariant distribution, it means the chain is not
reversible.

\begin{example*}
    Simple random walk on a graph. $G = (V, E)$,
    $E$ is edge set, $V$ is vertex set. $G$ is
    finite and connected.
    \[ P(x, y) = \begin{cases}
        \frac{1}{d(x)} & (x, y) \in E \\
        0 & \text{otherwise}
    \end{cases} \]
    $d(x)$ is the degree of $x$. Then since $G$ is
    connected, $P$ is irreducible. To find $\pi$,
    let's look at Detailed Balance equation
    \[ (x, y) \in E \quad \pi(x) \ub{P(x,
    y)}_{\frac{1}{d(x)}} = \pi(y) \ub{P(y,
    x)}_{\frac{1}{d(y)}} \]
    so $\pi$ must satisfy
    \[ \pi(x) \frac{1}{d(x)} = \pi(y)
    \frac{1}{d(y)} \quad \forall (x, y) \in E \]
    Taking $\nu(x) = d(x)$, then $\nu$ is an
    invariant measure. So
    \[ \pi(x) = \frac{d(x)}{\sum_{y \in V} d(y)} =
    \frac{d(x)}{2|E|} \]
    So simple random walk on $G$ is
    \emph{reversible}.
\end{example*}

\subsubsection*{Convergence to equilibrium}

\begin{theorem*}
    $I$ \emph{finite}, $i \in I$ such that
    $\forall j$
    \[ p_{ij}(n) \to \pi(j) \]
    as $n \to \infty$. Then $\pi$ is invariant.
\end{theorem*}

\noindent
$P$ has invariant distribution $\pi$. Question:
Under what conditions do we have convergence to
$\pi$?

\begin{example*}
    \begin{center}
        \includegraphics[width=0.6\linewidth]
        {images/eba5c42a60e411ed.png}
    \end{center}
    $p + q = 1$, $p = \frac{2}{3}$, $q =
    \frac{1}{3}$ $P^n(0, 0)$. If $n$ is odd, then
    \[ P^n(0, 0) = 0 \]
\end{example*}

\begin{flashcard}
\begin{definition*}
    $P$ transition matrix, $i \in I$. The period
    of $i$ is defined
    \[ d_i = \cloze{\gcd \{n \ge 1 : P^n(i, i) >
    0\}} \]
    $i$ is called \emph{aperiodic} if \cloze{$d_i
    = 1$.}
\end{definition*}
\end{flashcard}

\begin{flashcard}[period-1-markov-chain-condition]
\begin{lemma*}
    Let $P$ be a transition matrix and $i \in I$.
    Then $d_i = 1$ if and only if
    \cloze{\[ P^n(i, i) > 0 \]
    for all $n$ sufficiently large.}
\end{lemma*}
\end{flashcard}

\begin{proof}
    \begin{enumerate}
        \item[$\Leftarrow$] Obvious.
        \item[$\Rightarrow$] If $d_i = 1$ then
            want to show
            $P^n(i, i) > 0$ for all $n$ large
            enough.
            \[ D(i) = \{n \ge 1 : P^n(i, i) > 0\} \]
            Observation: if $n, m \in D(i)$ then
            $n + m \in D(i)$. So suffices to prove
            that $D(i)$ contains 2 consecutive
            integers. Say it contains $m, m + 1$.
            Then by the observation it will also
            contain $am + b(m + 1)$ for all $a, b
            \in \NN$. One can check that
            \[ D(i) \supset \{n : n \ge m^2\} \]
            Suppose $\min\{x - y : x > y, x, y \in
            D(i)\} = r \ge 2$. Let $n, m \in
            D(i)$ such that $n = m + r$. Then
            there exists $k = lr + s$ with $0 < s
            < r$, $l \in \NN$ such that $k \in
            D(i)$. If there does not exist such
            $k$, then all elements of $D(i)$ would
            be multiples of $r$, and hence gcd
            $D(i)$ would be $r$, contradiction. \\
            Let $a = (l + 1)n$ and $b = (l + 1)m +
            k$. By observation, $a, b \in D(i)$.
            \[ a - b = r - s < r \]
            so $r = 1$, so $D(i)$ contains 2
            consecutive integers.
    \end{enumerate}
\end{proof}

\begin{lemma*}
    If $P$ is irreducible and $i \in I$ is
    aperiodic, then all states are aperiodic.
\end{lemma*}

\begin{proof}
    Let $j \in I$. There exists $r, s \ge 0$ such
    that
    \[ P^r(i, j) > 0 \text{ and } P^s(j, i) > 0 \]
    (by irreducibility)
    \[ P^{n + r + s}(j, j) \ge P^s(j, i) P^n(i, i)
    P^r(i, j) > 0 \]
    for all $n$ sufficiently large.
\end{proof}