% vim: tw=50 % 08/11/2022 10AM \noindent Recall that \[ \tau_k = \inf\{n \ge 1 : X_n = k\} \] $k$ is recurrent if and only if $\PP_k(\tau_k < \infty) = 1$. $k$ is positive recurrent if $\EE_k[ \tau_k] < \infty$. Otherwise $k$ is null-recurrent. \begin{flashcard}[positive-recurrent-states-equivalences] \begin{theorem*} Let $P$ be an irreducible matrix. Then the following are equivalent: \begin{enumerate}[(1)] \item All states are positive recurrent \item \cloze{Some state is positive recurrent} \item \cloze{There exists an invariant distribution $\pi$.} \end{enumerate} If any of the above holds, then \[ \cloze{\pi_k = \frac{1}{\EE_k[\tau_k]}} \] \end{theorem*} \end{flashcard} \begin{proof} \phantom{} \begin{enumerate} \item[(1) $\implies$ (2)] Obvious. \item[(2) $\implies$ (3)] Let $k$ be the positive recurrent state. \[ \forall i \quad \nu_k(i) = \EE_k \left[ \sum_{l = 0}^{\tau_k - 1} \mathbbm{1}(X_l = i) \right] \] implies $k$ is also recurrent so by theorem from last time, $\nu_k P = \nu_k$. $\nu_k$: invariant measure. \[ \sum_{i \in I} \nu_k(i) = \EE_k \left[ \sum_{l = 0}^{\tau_k - 1} \sum_{i \in I} \mathbbm{1}(X_l = i) \right] = \EE_k[\tau_k] \] Since $k$ is positive recurrent, implies $\EE_k[\tau_k] < \infty$. So we can define \[ \pi_i = \frac{\nu_k(i)}{\EE_k[\tau_k]} \] invariant distribution. \item[(3) $\implies$ (1)] Let $\pi$ be the invariant distribution. Let $k$ be a state. Need to show $k$ is positive recurrent. First show $\pi_k > 0$. Exists $i \in I$ such that $\pi_i > 0$. $\pi = \pi P = \pi P^n$ for all $n$. \[ \pi_k = \sum_k \pi_j P^n(j, k) \] Take $n$ such that $P^n(i, k) > 0$ (irreducibility of $P$). Then \[ \pi_k \ge \pi_i P^n(i, k) > 0 \] Define $\lambda_i = \frac{\pi_i}{\pi_k}$: invariant measure, with $\lambda_k = 1$. So since $P$ is irreducible $\lambda \ge \nu_k$, i.e. $\forall i$ $\nu_k(i) \le \lambda_i$. \[ \EE_K[\tau_k] = \sum_{i \in I} \nu_k(i) \le \sum_{i \in I} \lambda_i = \frac{1}{\pi_k} \] So $\EE_k[\tau_k] \le \frac{1}{\pi_k} < \infty$. So $k$ is positive recurrent. \end{enumerate} Suppose (1), (2), (3) hold. Let $k$ be a state. Then $k$ is positive recurrent. Define $\lambda_i = \frac{\pi_i}{\pi_k}$: invariant measure with $\lambda_k = 1$. Since $P$ is recurrent, $\lambda = \nu_k$ (that is, $\lambda_i = \nu_k(i)$ for all $i$). So \[ \sum_{i \in I} \lambda_i = \sum_{i \in I} \nu_k(i) \] so \[ \frac{1}{\pi_k} = \EE_k[\tau_k] \] \end{proof} \begin{flashcard}[expected-hitting-number-in-terms-of-inv-dist] \begin{corollary*} $P$ irreducible, $\pi$ invariant distribution. Then for all $x, y$, $\nu_x(y) = \cloze{\frac{\pi(y)}{\pi(x)}}$. \end{corollary*} \end{flashcard} \begin{example*} Simple random walk on $\ZZ$. $P(x, x + 1) = P(x, x - 1) = \half$. $P$ is recurrent ($d = 1$). Does there exist an invariant distribution? $\pi = \pi P$. Need \[ \pi_i = \half \pi_{i - 1} + \half \pi_{i + 1} \] Then $\pi = 1$ for all $i$ satisfies $\pi = \pi P$. Since $P$ is recurrent, all invariant measures have to be multiples of $\pi_i = 1$ for all $i$. So there does not exist invariant distribution, so not positive recurrent. \end{example*} \begin{example*} $\ZZ$, $P(x, x + 1) = p$, $P(x, x - 1) = q$, $p + q = 1$, $p > q$. Need $\pi = \pi P$, need \[ \pi_i = p \pi_{i - 1} + q\pi_{i + 1} \] Solve to get \[ \pi = a + b \left( \frac{p}{q} \right)^i \] is an invariant measure for any choice of $a, b$. So no uniqueness up to multiplicative factors. Indeed, $P$ is transient. \end{example*} \begin{example*} Simple random walk on $\ZZ^3$ transient. $\pi_i = 1$ for all $i \in \ZZ^3$ invariant measure. This shows that the existence of an invariant \emph{measure} does not imply recurrence. \end{example*} \begin{example*} $\ZZ_+$, $P(x, x + 1) = p$, $P(x, x - 1) = q$, $p + q = 1$, $p < q$. $P(0, 1) = p$, $P(0, 0) = q$. Look for $\pi$ such that $\pi = \pi P$. \[ \pi_i = p \pi_{i - 1} + q \pi_{i + 1} \quad i \ge 1 \] \[ \pi_0 = q \pi_1 + \pi_0 q \] \[ \pi_1 = \pi_0 \frac{p}{q}, \quad \pi_i = \left( \frac{p}{q} \right)^i \pi_0 \quad \forall i \ge 1 \] $p < q$, set $\pi_0 = 1 - \frac{p}{q}$, to get \[ \pi_i = \left( \frac{p}{q} \right)^i \left( 1 - \frac{p}{q} \right), \quad i \ge 0 \] so there exists an invariant distribution, so positive recurrent. \end{example*} \subsubsection*{Time Reversibility} \begin{flashcard}[time-reversing-matrix] \begin{proposition*} $P$ irreducible, $\pi$ invariant distribution. Fix $N \in \NN$ and $X_0 \sim \pi$. Define $Y_n = X_{N - n}$, $0 \le n \le N$. Then $(Y_n)_{0 \le n \le N}$ is a Markov chain with transition matrix \[ \hat{P}(x, y) = \cloze{\frac{\pi(y)}{\pi(x)} P(y, x)} \] and $\pi$ is an invariant distribution. If $P$ is irreducible, then $\hat{P}$ is too. \end{proposition*} \end{flashcard} \begin{proof} $\hat{P}$ is a transition matrix, since \[ \sum_y \hat{P}(x, y) = \sum_y \frac{\pi(y)}{\pi(x)} P(y, x) = \frac{\pi(x)}{\pi(x)} = 1 \] Let $y_0, \dots, y_N \in I$. Then \begin{align*} \PP(Y_0 = y_0, \dots, Y_N = y_N) &= \PP(X_n = y_0, \dots, X_0 = y_N) \\ &= \PP(X_0 = y_N, \dots, X_n = y_0) \\ &= \pi(y_N) P(y_N, y_{N - 1}) \cdot P(y_1, y_0) \\ &= \pi(y_{N - 1}) \hat{P}(y_{N - 1}, y_N) P(y_{N - 1}, y_{N - 2}) \cdots P(y_1, y_0) \\ &= \cdots \\ &= \pi(y_0) \hat{P}(y_0, y_1) \cdots \hat{P}(y_{N - 1}, y_N) \end{align*} so $Y$ is $\Markov(\pi, \hat{P})$. Check $\hat{P}$ has invariant distribution $\pi$. Need to show $\pi \hat{P} = \pi$. \[ \sum_x \pi(x) \hat{P}(x, y) = \sum_x \pi(x) \frac{\pi(y)}{\pi(x)} P(y, x) = \pi(y) \] so $\pi \hat{P} = \pi$. \end{proof} \myskip Also, $\hat{P}$ is irreducible.