% vim: tw=50
% 18/10/2022 10AM

\subsubsection*{Communicating classes}

\begin{flashcard}
\begin{definition*}[Communicating classes]
    Let $X$ be a Markov Chain with matrix $P$ on
    $I$. Let $x, y \in I$. We say $x \to y$ ($x$
    leads to $y$) if
    \[ \cloze{\PP_x(X_m = y \text{ for some } m \ge 0) >
    0 .} \]
    We say that $x$ and $y$ communicate and $x
    \comm y$ if both $x \to y$ and $y
    \to x$.
\end{definition*}
\end{flashcard}

\begin{theorem*}
    The following are equivalent:
    \begin{enumerate}[(1)]
        \item $x \to y$
        \item $\exists$ a sequence $x = x_0, x_1,
            \dots, x_k = y$ such that
            \[ P(x_0, x_1) \cdots P(x_{k - 1},
            x_k) > 0 \]
        \item $\exists n \ge 0$ such that
            $p_{xy}(n) > 0$ (recall that
            $p_{xy}(n)$ is the $(x, y)$ element of
            $P^n$, and is also equal to $\PP_x(X_n
            = y)$)
    \end{enumerate}
\end{theorem*}

\begin{proof}
    First we prove (1) $\iff$ (3). We have:
    \[ \{X_n = y \text{ for some $n \ge 0$}\} =
    \bigcup_{n \ge 0} \{X_n = y\} \]
    If $x \to y$, then $\exists n \ge 0$ such that
    $\PP_x(X_n = y) > 0$. From the definition of
    $\to$, we immediately have (3) $\implies$ (1).
    \myskip
    Now we prove (2) $\iff$ (3):
    \[ \PP_x(X_n = y) = \sum_{x_1, \dots, x_{n -
    1}} p(x, x_1) \cdots p(x_{n - 1}, y) \]
    so (2) $\iff$ (3).
\end{proof}

\begin{corollary*}
    $\comm$ defines an equivalence
    relation on $I$.
\end{corollary*}

\begin{proof}
    $x \comm x$, because $p_{xx}(0) =
    1$. Transitivity: suppose $X \comm
    y$ and $y \comm z$. Then from (2),
    $x \comm z$.
\end{proof}

\begin{definition*}
    The equivalence classes induced by $\comm$ on
    $I$ are called communicating classes. We say
    that a class $C$ is \emph{closed} if whenever
    $x \in C$ and $x \to y$, then $y \in C$.
\end{definition*}

\begin{hiddenflashcard}
    \begin{definition*}[Closed communicating class]
        We say that a communicating class $C$ is
        \emph{closed} \cloze{whenever $x \in C$ and
        $x \to y$, then $y \in C$.}
    \end{definition*}
\end{hiddenflashcard}

\begin{flashcard}
\begin{definition*}
    A matrix $P$ (transition) is called
    \emph{irreducible} if \cloze{it has a single
    communicating class.}\fcscrap{ In other words,
    $x \comm y$ for all $x, y \in I$.}
\end{definition*}
\end{flashcard}

\begin{definition*}
    A state $x$ is called \emph{absorbing} if $\{x\}$
    is a closed class. Equivalently if the Markov
    chain started from $x$ then it stays at $x$
    forever.
\end{definition*}

\begin{definition*}
    $A \subseteq U$. $\tau_A : \Omega \to \NN \cup
    \{\infty\}$
    \[ \tau_A = \inf\{n \ge 0 : X_n(\omega) \in
    A\} \]
\end{definition*}

\noindent
Convention: $\inf(\emptyset) = \infty$. $\tau_A$
is the first hitting time of $A$. \myskip
Denote $h_i^A = \PP_i(\tau_A < \infty)$, $i \in
I$. \\
$h^A : I \to [0, 1]$, ($h_i^A : i \in I$) is
vector of hitting probability. \myskip
Also define $k^A : I \to \RR_+ \cup \{\infty\}$,
the mean hitting time. So
\[ k_i^A = \EE_i[\tau_A] = \sum_{n = 1}^\infty n
\PP_i(\tau_A = n) + \ub{\infty \cdot \PP_i(\tau_A =
\infty)}_{0 \cdot \infty = 0} \]

\begin{center}
    \includegraphics[width=0.6\linewidth]
    {images/f3726f2a4ec711ed.png}
\end{center}
$\PP_2(\tau_4 < \infty) = h_2^{\{4\}}$. $\tau_4 =
\tau_{\{4\}}$.
\[ h_2 = \half h_3 + \half h_1 \]
\[ h_3 = \half + \half h_2 \]
\[ \implies h_2 = \frac{1}{3} \]
($h_1 = 0$, $h_4 = 1$)
$k_2^{\{1, 4\}} = \EE_2[\tau_{\{1, 4\}}]$
\[ k_2 = 1 + \half \cdot 0 + \half k_3 \]
\[ k_3 = 1 + \half \cdot 0 + \half k_2 \]
\[ \implies k_2 = 2 \]
$k_1 = k_4 = 0$.

\begin{flashcard}
\begin{theorem*}
    Let $A \subseteq I$. The vector $(h_i^A : i
    \in I)$ is \cloze{a solution to the linear
    system}
    \[ \cloze{h_i^A = \begin{cases}
        1 & \text{if $i \in A$} \\
        \sum_j P(i, j) h_j^A & i \not\in A
    \end{cases}} \]
    \cloze{The vector $(h_i^A)$ is the minimal
    non-negative solution to this system.} \\
    \fcscrap{A solution $(h_i^A)$ is minimal if for any other
    non-negative solution $(X_i)$, we have that
    $h_i^A \le X_i ~ \forall i$.}
\end{theorem*}
\end{flashcard}

\begin{proof}
    Clearly, if $i \in A$, then $h_I^A = 1$.
    Assume $i \not\in A$.
    \[ h_i^A = \PP_i(\tau_A < \infty) \]
    \[ \{\tau_A < \infty\} = \bigcup_{n =
    0}^\infty \{\tau_A = n\} = \bigcup_{n =
    0}^\infty \{X_0 \not\in A, X_1 \not\in A,
    \dots, X_{n - 1} \not\in A, X_n \in A\} \]
    \begin{align*}
        \PP_i(\tau_A < \infty)
        &= \sum_{n = 1}^\infty \PP_i(X_1 \not\in
        A, \dots, X_{n - 1} \not\in A, X_n \in A)
        \\
        &= \PP_i(X_1 \in A) + \sum_{n = 2}^\infty
        \PP_i(X_1 \not\in A, \dots, X_{n - 1}
        \not\in A, X_n \in A)
    \end{align*}
    Now compute:
    \begin{align*}
        \PP_i(X_1 \not\in A, \dots, X_{n - 1}
        \not\in A, X_n \in A)
        &= \sum_{j \not\in A} \PP_i(X_1 = j, X_2
        \not\in A, \dots, X_{n - 1} \not\in A, X_n
        \in A) \\
        &= \sum_{j \not\in A} \PP_i(X_2 \not\in A,
        \dots,X_{n - 1} \not\in A, X_n \in A \mid
        X_0 = i, X_1 = j) P(i, j) \\
        &= \sum_{j \not\in A} P(i, j) \PP_j(X_1
        \not\in A, \dots, X_{n - 2} \not\in A,
        X_{n - 1} \in A)
    \end{align*}
    Now plus back in:
    \begin{align*}
        h_i^A
        &= \PP_i(X_1 \in A) + \sum_{n = 1}^\infty
        \sum_{j \not\in A} P(i, j) \ub{\PP_j(X_1
        \not\in A, \dots, X_n \in A)}_{h_j^A} \\
        &= \sum_{j \in A} P(i, j) \ub{h_j^A}_{=1} + \sum_{j
        \not\in A} P(i, j) h_j^A \\
        \implies h_i^A
        &= \sum_j P(i, j) h_j^A
    \end{align*}
    So $h_i^A$ is a solution as claimed. \myskip
    Now we prove minimality. Let $(x_i)$ be
    another non-negative solution. Need to show
    that $h_i^A \le x_i$ for all $i$. If $i
    \not\in A$, then
    \[ x_i = \sum_j P(i, j) x_j \]
    \[ \implies x_i = \sum_{j \in A} P(i, j) +
    \sum_{j \not\in A} P(i, j) x_j \]
    \begin{align*}
        x_i
        &= \sum_{j \in A} P(i, j) + \sum_{j
        \not\in A} \sum_{j \in A} P(i, j)P(j, k) +
        \sum_{j \not\in A} \sum_{k \not\in A}
        P(i, j)P(j, k) x_k \\
        x_i
        &= \PP_i(X_1 \in A) + \PP_i(X_1 \not\in A,
        X_2 \in A) + \sum_{j \not\in A} \sum_{k
        \not\in A} P(i, j)P(j, k) x_k \\
        x_i
        &\ge \PP_i(X_1 \in A) + \PP_u(X_1 \not\in
        A, X_2 \not\in A) + \cdots
        + \PP_i(X_1 \not\in A, \dots, X_{n - 1}
        \not\in A, X_n \in A)
    \end{align*}
    (the inequality holds because the remaining
    terms are all non-negative since we assume
    that the $x_i$ are non-negative). Rewriting:
    \[ x_i \ge \PP_i(\tau_A \le n) \quad \forall n
    \in \NN .\]
    \[ \{\tau_A \le n\} \nearrow \bigcup_n
    \{\tau_A \le n\} = \{\tau_A < \infty\} \]
    so $\PP_i(\tau_A \le n) \nearrow \PP_i(\tau_A
    < \infty)$ hence $x_i \ge \PP_i(\tau_A <
    \infty) = h_i^A$.
\end{proof}