% vim: tw=50
% 13/10/2022 10AM

\begin{flashcard}
\begin{theorem*}[Simple Markov property]
    Suppose $X$ is $\Markov(\lambda, P)$ with
    values in $I$. Let \cloze{$m \in \NN$ and $i
    \in I$.}
    Then \cloze{conditional on $X_m = i$, the process
    $(X_{m + n})_{n \ge 0}$ is $\Markov(\delta_i,
    P)$ and it is independent of $X_0, \dots,
    X_m$.}
\end{theorem*}
\end{flashcard}

\begin{proof}
    Let $x_0, x_1, \dots, x_n \in I$.
    \begin{align*}
        &\phantom{{}={}} \PP(X_m = x_0, X_{m + 1} = x_1,
        \dots, X_{m + n} = x_n \mid X_m = i) \\
        &= \mathbbm{1}_{i = x_0} \frac{\PP(X_m =
        x_0, \dots, X_{m + n} = x_n}{\PP(X_m = i)}
        \tag{$*$}
    \end{align*}
    \begin{align*}
        &\phantom{{}={}} \PP(X_m = x_0, \dots, X_{m + n}
        = x_n) \\
        &= \sum_{y_0, \dots, y_{m - 1}} \PP(X_0 =
        y_0, \dots, X_{m - 1} = y_{m - 1}, X_m =
        x_0, \dots, X_{m + n} = x_n) \\
        &= \sum_{y_0, \dots, y_{m - 1}}
        \lambda(y_0) P(y_0, y_1) \cdots P(y_{m -
        2}, y_{m - 1})P(y_{m - 1}, x_0) \cdots
        P(x_{n - 1}, x_n) \\
        &= p(x_0, x_1) \cdots P(x_{n - 1}, x_n)
        \ub{\sum_{y_0, \dots, y_{m - 1}} \lambda(y_0)
        P(y_0, y_1) \cdots P(y_{m - 1},
        \lambda_0)}_{= \PP(X_m = i)}
    \end{align*}
    putting back into ($*$) we get that
    \[ \mathbbm{1}_{i = x_0} P(x_0, x_1) \cdots
    P(x_{n - 1}, x_n) \implies \Markov(\delta_i,
    P) \]
    ($\mathbbm{1}_{i = x_0}$ is another notation
    for $\delta_{ix_0}$). Let $m \le i_1 < i_2 <
    \cdots < i_k$, $y_0 = i$. Then
    \begin{align*}
        &\phantom{{}={}} \PP(X_{i_1} = x_1, \dots,
        X_{i_k} = x_k, X_0 = y_0, \dots, X_m = y_m
        \mid X_m = i) \\
        &= \frac{\PP(X_{i_1} = x_1, \dots, X_{i_k}
        = x_k, X_0 = y_0, \dots, X_m =
        y_m)}{\PP(X_m = i)} \\
        &= \frac{\lambda(y_0) P(y_0, y_1) \cdots
        P(y_{m - 1}, y_m)}{\PP(X_m = i)}
        \PP(X_{i_1} = x_1, \dots, X_{i_k} = x_k
        \mid X_m = i) \\
        &= \PP(X_{i_1} = x_1, \dots, X_{i_k} = x_k
        \mid X_m = i) \PP(X_0 = y_0, \dots, X_m =
        y_m \mid X_m = i)
    \end{align*}
\end{proof}

\myskip
$X \sim \Markov(\lambda, P)$
\begin{align*}
    \PP(X_n = x)
    &= \sum_{x_0, \dots, x_{n - 1}} \PP(X_0 = x_0,
    \dots, X_{n - 1} = x_{n - 1}, X_n = x) \\
    &= \sum_{x_0, \dots, x_{n - 1}} \lambda(x_0)
    P(x_0, x_1) \cdots P(x_{n - 1}, x) \\
    &= (\lambda P^n)_x
\end{align*}
By convention $P^0 = I$.
\[ \PP(X_{n + m} = y \mid X_m = x) \]
Conditional on $X_m = x$, $(X_{m + n})_{n \ge 0}$
is $\Markov(\delta_x, P)$. So
\[ \PP(X_{n + m} = y \mid X_m = x) = (\delta_x
P^n)_y = (P^n)_{xy} \]
We will write
\[ p_{xy}(n) = (P^n)_{xy} \]
Let $A$ be such an event. We will write
\[ \PP_i(A) = \PP(A \mid X_0 = i) \]

\subsubsection*{Examples for $P^n$}

Consider
\[ P =
\begin{pmatrix}
    1 - \alpha & \alpha \\
    \beta & 1 - \beta
\end{pmatrix}
\]
\[ P^{n + 1} = P^n \cdot P = P \cdot P^n \]
So
\[ p_{11}(n + 1) = (1 - \alpha) p_{11}(n) +
p_{12}(n) \]
\[ p_{11}(n) + p_{12}(n) = 1 \qquad p_{11}(0) = 1 \]
\[ p_{11}(n) = \begin{cases}
    \frac{\alpha}{\alpha + \beta} +
    \frac{\alpha}{\alpha + \beta} (1 - \alpha -
    \beta)^n & \text{if $\alpha + \beta > 0$} \\
    1 & \text{if $\alpha + \beta = 0$}
\end{cases} \]
In this simple case, it is easy to solve directly
for $P^n$. However this is not generally the case
for large matrices. \myskip
Finding eigenvalues of $P$ is another useful
method. Let $P$ be a $k \times k$ stochastic
matrix. Let $\lambda_1, \dots, \lambda_k$ be the
eigenvalues of $P$.
\begin{itemize}
    \item If $\lambda_1, \dots, \lambda_k$ are all
        distinct, then $P$ is diagonalisable.
        \[ P = UDU^{-1} \implies P^n = UD^n U^{-1} \]
        \[ p_{11}(n) = \alpha_1 \lambda_1^n +
        \alpha_2 \lambda_2^n + \cdots + \alpha_k
        \lambda_k^n \]
        $p_{11}(0) = 1$. Plug in small values of
        $n$, then solve the system to find
        $\alpha_1, \dots, \alpha_k$. If one of the
        eigenvalues is complex, say $\lambda_{k -
        1}$, then also its conjugate will be an
        eigenvalue say $\lambda_k = \ol{\lambda_{k
        - 1}}$.
        \[ \lambda_{k - 1} = re^{i\theta} =
        r\cos\theta + ir\sin\theta \]
        \[ \lambda_k = r\cos\theta - ir\sin\theta \]
        It becomes easier (calculations) to write
        the general form as
        \[ p_{11}(n) = \alpha_1 \lambda_1^n +
        \cdots + \alpha_{k - 2} \lambda_{k - 2}^n
        + \alpha_{k - 1} r^n \cos(n\theta) +
        \alpha_k r^n \cos(n\theta) \]
    \item If the eigenvalues are not all distinct
        then suppose $\lambda$ appears with
        multiplicity 2. Then we also include the
        term $\alpha n + \beta) \lambda^n$ in the
        expression for $p_{11}(n)$. (Jordan normal
        form).
\end{itemize}
\[ P =
    \begin{pmatrix}
        0 & 1 & 0 \\
        0 & \half & \half \\
        \half & 0 & \half
    \end{pmatrix}
\]
eigenvalues : $1, \frac{i}{2}, -\frac{i}{2}$.
\[ p_{11}(n) = \alpha_1 + \alpha_2 \left( \half
\right)^n \cos \left( \frac{n\pi}{2} \right) +
\alpha_3 \left( \half \right)^n \sin \left(
\frac{n\pi}{2} \right) \]
\[ p_{11}(0) = 1 \qquad p_{11} = 0 \qquad
p_{11}(2) = 0 \]
\[ p_{11}(n) = \frac{1}{5} + \left( \half
\right)^n \left( \frac{4}{5} \cos \left(
\frac{n\pi}{2} \right) - \frac{2}{5} \sin \left(
\frac{n\pi}{2} \right) \right) \]