% vim: tw=50
% 03/11/2022 10AM

\begin{hiddenflashcard}
    What does $\nu_k(i)$ represent?
    \[ \nu_k(i) = \cloze{\EE_k \left[ \sum_{l =
    0}^{\tau_k - 1} \mathbbm{1}(X_l = i) \right]} \]
\end{hiddenflashcard}

\begin{flashcard}[nu-k-invariant-measure]
\begin{theorem*}
    If $P$ is irreducible and recurrent, then
    $\nu_k$ is \cloze{an invariant
    measure\fcscrap{ ($\nu_k =
    \nu_k P$)} satisfying
    \[ 0 < \nu_k(i) < \infty ~\forall i \prompt{\qquad
    \text{and} \qquad \nu_k(k) = 1} \]}\fcscrap{and
    \[ \nu_k(k) = 1 \]}
    \prompt{\\[-2\baselineskip]}
\end{theorem*}
\end{flashcard}

\begin{proof}
    Obviously $\nu_k(k) = 1$. Let $i \in I$. We
    will prove
    \[ \nu_k(i) = \sum_j P(j, i) \nu_k(j) \]
    By recurrence we get $\tau_k < \infty$ with
    probability 1 and $X_{T_k} = k$. So
    \begin{align*}
        \nu_k(i)
        &= \EE_k \left[ \sum_{l = 1}^{T_k}
        \mathbbm{1}(X_l = i) \right] \\
        &= \EE_k \left[ \sum_{l = 1}^\infty
        \mathbbm{1}(X_l = i) \mathbbm{1}(\tau_k
        \ge 1) \right] \\
        &= \sum_{l = 1}^\infty \PP_k(X_l = i,
        \tau_k \ge l) \\
        &= \sum_{l = 1}^\infty \sum_j \PP_k (X_l =
        i, X_{l - 1} = j, \tau_k \ge l) \\
        &= \sum_{l = 1}^\infty \sum_j \PP_k(X_l =
        i \mid X_{l - 1} = j, \tau_k \ge l)
        \PP_k(X_{l - 1} > j, \tau_k \ge l)
    \end{align*}
    \[ \{\tau_k \ge l\} = \{T_k \le l - 1\}^c \]
    By the Markov property
    \begin{align*}
        \PP_k(X_l = i \mid X_{l - 1} = j, \tau_k
        \ge l)
        &= \PP_k( \tau_l = i \mid X_{l - 1} = j)
        \\
        &= P(j, i)
    \end{align*}
    so
    \begin{align*}
        \nu_k(i)
        &= \sum_{l = 1}^\infty \sum_j P(j, i)
        \PP_k(X_{l - 1} = j, \tau_k \ge l) \\
        &= \sum_j P(j, i) \EE_k \left[ \sum_{l =
        1}^\infty \mathbbm{1}(X_{l - 1} = j,
        \tau_k \ge l) \right] \\
        &= \sum_j P(j, i) \EE_k \left[ \sum_{l =
        0}^{\tau_k - 1} \mathbbm{1}(X_l = j)
        \right] \\
        \implies \nu_k(i)
        &= \sum_j \nu_k(j) P(j, i)
    \end{align*}
    for all $i$. So since $\nu_k(i) > 0$:
    \[ \nu_k = \nu_k P^m \]
    for all $m$
    \[ \nu_k(i) \ge \nu_k(k) P_{ki}(m) = P_{ki}(m) \]
    By irreducibility there exists $m$ such that
    $p_{ki}(m) > 0$ so $\nu_k(i) > 0$.
    \[ \nu_k(k) = \sum_j \nu_k(j) p_{jk}(m) \]
    \[ 1 = \nu_k(k) \ge \nu_k(i) p_{ik}(m) \]
    Take $n$ such that $p_{ik}(n) > 0$
    (irreducibility of $P$) then we get
    \[ \nu_k(i) \le \frac{1}{p_{ik}(n)} < \infty
    \qedhere \]
\end{proof}

\begin{theorem*}
    If $P$ is irreducible and $\lambda$ is an
    invariant measure satisfying $\lambda_k = 1$,
    then
    \[ \lambda \ge \nu_k \quad (\forall i
    ~\lambda_I \ge \nu_k(i)) \]
    If $P$ is also recurrent, then $\lambda =
    \nu_k$.
\end{theorem*}

\begin{proof}
    $\lambda_i \ge 0$ for all $i$.
    \begin{align*}
        \lambda_i
        &= \sum_j \lambda_j P(j, i) \\
        &= P(k, i) + \sum_{j_1 \neq k} P(j, i)
        \lambda_{j_1} \\
        &= P(k, i) + \sum_{j_1 \neq k} P(k, j_1)
        P(j_1, 1) + \sum_{\substack{j_1 \neq
        k\\j_2 \neq k}} P(j_2, j_1)P(j_1, i)
        \lambda_{j_2} \\
        &= P(k, i) + \sum_{j_1 \neq k} P(k, j_1)
        P(j_1, i) + \cdots + \sum_{j_1, \dots,
        j_{n - 1} \neq q} P(k, j_{n - 1}) \cdots
        P(j_1, i) \\
        &~~~~~+ \sum_{j_1, \dots, j_n \neq k}
        P(j_n, j_{n - 1}) \cdots P(j_1, i)
        \lambda_{j_n}
    \end{align*}
    So
    \[ \lambda_i \ge \PP_k(X_1 = i, \tau_k \ge 2)
    + \PP_k(X_2 = i, \tau_k \ge 3) + \cdots +
    \PP_k(X_n = i, \tau_k \ge n + 1) \]
    so
    \begin{align*}
        \lambda_i
        &\ge \EE_k \left[ \sum_{l = 1}^n
        \mathbbm{1}(X_l = i, \tau_k \ge l + 1)
        \right] \\
        &= \EE-k \left[ \sum_{l = 0}^n
        \mathbbm{1}(X_l = i, l \le \tau_k - 1)
        \right] \\
        &= \sum_{l = 0}^n \EE_k [\mathbbm{1}(X_l
        = i, l \le \tau_k - 1)] \\
        &\to \sum_{l = 0}^\infty \EE_k
        [\mathbbm{1}(X_l = i, l \le \tau_k - 1)]
        \\
        &= \nu_k(i)
    \end{align*}
    $\lambda_i \ge \nu_k(i)$ for all $i$. \\
    If $P$ is recurrent, then $\nu_k$ is an
    invariant measure with $\nu_k(k) = 1$. So we also
    have that $\mu_i = \lambda_i - \nu_k(i)$ is an
    invariant measure, since we know $\lambda_i
    \ge \nu_k(i)$, hence $\mu_i \ge 0$. Need to
    show that $\mu_i = 0$ for all $i$. Let $i \in
    I$.
    \[ 0 = \mu_k = \sum_j \mu_i P^m (j, k) \quad
    \forall m \]
    \[ \implies \mu_k \ge \mu_i P^m (i, k) \]
    Take $m$ such that $p_m(i, k) > 0$. Then
    $\mu_i = 0$.
\end{proof}

\begin{remark*}
    If $P$ is irreducible and recurrent, then all
    invariant measures are unique up to
    multiplicative factors.
\end{remark*}

\noindent
Question: When can we get an invariant
distribution $\pi = \pi P$, $\sum \pi_i = 1$?
\myskip
Let $P$ be irreducible and recurrent. By the
uniqueness (up to multiplication) we can get an
invariant distribution (unique) if
\[ \sum_{i \in I} \nu_k(i) < \infty \]
Then
\[ \pi_I = \frac{\nu_k(i)}{\sum_j \nu_k(j)} \]
\begin{align*}
    \sum_{i \in I} \nu_k(i)
    &= \sum_{i \in I} \EE_k \left[ \sum_{l =
    0}^{\tau_k - 1} \mathbbm{1}(X_l = i) \right]
    \\
    &= \EE_k \left[ \sum_{l = 0}^{\tau_k - 1}
    \sum_{i \in I} \mathbbm{1}(X_l = i) \right] \\
    &= \EE_k[\tau_k]
\end{align*}
If $\EE_k[\tau_k] < \infty$, then we can
normalise.

\begin{flashcard}[positive-null-recurrent]
\begin{definition*}
    Let $P$ be irreducible and recurrent\fcscrap{, $\tau_k
    = \inf\{n \ge 1 : X_n = k\}$, $\PP_k(\tau_k <
    \infty) = 1$ for all $k$}. We say $k$ is
    \emph{positive recurrent} if
    \[ \cloze{\EE_k[\tau_k] < \infty} \]
    We say $k$ is \emph{null recurrent} if
    \[ \cloze{\EE_k[\tau_k] = \infty} \]
    \fcscrap{If $k$ is positive recurrent, then
    \[ \pi_k = \frac{\nu_k(k)}{\EE_k [\tau_k]} =
    \frac{1}{\EE_k [\tau_k]} \]}
\end{definition*}
\end{flashcard}