%! TEX root = EMC.tex
% vim: tw=50
% 28/01/2025 09AM

\begin{fcprop}[]
  \label{prop:1.7}
  Assuming:
  - $X$ is uniformly distributed on a set $A$ of
  size $n$
  Then:
  $\ent X = \log n$.
\end{fcprop}

Reminder: $\log$ here is to the base $2$ (which is
the convention for this course).

\begin{proof}
  Let $r$ be a positive integer and let $X_1,
  \ldots, X_r$ be independent copies of $X$.

  Then $(X_1, \ldots, X_r)$ is uniform on $A^r$
  and
  \[
    \ent{X_1, \ldots, X_r} = r\ent X
  .\]
  Now pick $k$ such that $2^k \le n^r \le 2^{k +
  1}$. Then by \gls{invariance}, \gls{maximality},
  and \cref{prop:1.6}, we have that
  \[
    k \le r \ent X \le k + 1
  .\]
  So
  \[
    \frac{k}{r} \le \log n \le \frac{k + 1}{r}
    \implies
    \frac{k}{r} \le \ent X \le \frac{k + 1}{r}
    \qquad \forall k, r
  \]
  Therefore, $\ent X = \log n$ as claimed.
\end{proof}

\begin{notation*}
  We will write $p_a = \Pbb[X = a]$.

  We will also use the notation $[n] = \{1, 2,
  \ldots, n\}$.
\end{notation*}

\begin{fcthm}[Khinchin]
  \label{thm:1.8}
  Assuming:
  - $H$ satisfies the Khinchin axioms
  - $X$ takes values in a finite set $A$
  Then:
  \[
    \ent X = \sum_{a \in A} p_a \log \left(
    \frac{1}{p_a} \right)
  .\]
\end{fcthm}

\begin{proof}
  First we do the case where all $p_a$ are
  rational (and then can finish easily by the
  \gls{continuity} axiom).

  Pick $n \in \Nbb$ such that for all $a$, there
  is some $m_a \in \Nbb \cup \{0\}$ such that $p_a =
  \frac{m_a}{n}$.

  Let $Z$ be uniform on $[n]$. Let $(E_a : a \in
  A)$ be a partition of $[n]$ into sets with
  $|E_a| = m_a$. By \gls{invariance} we may assume
  that $X = a \iff Z \in E_a$. Then
  \begin{align*}
    \log n
    &= \ent Z \\
    &= \ent{Z, X} \\
    &= \ent X + \cent ZX \\
    &= \ent X + \sum_{a \in A} p_a \cent Z{X =
    a} \\
    &= \ent X + \sum_{a \in A} p_a \log(m_a) \\
    &= \ent X + \sum_{a \in A} p_a (\log p_a +
    \log n)
  \end{align*}
  Hence
  \[
    \ent X = -\sum_{a \in A} p_a \log p_a
    = \sum_{a \in A} p_a \log \left( \frac{1}{p_a} \right)
  .\]
  By \gls{continuity}, since this holds if all $p_a$ are
  rational, we conclude that the formula holds in
  general.
\end{proof}

\begin{fccoro}[]
  \label{coro:1.9}
  Assuming:
  - $X$ and $Y$ random variables
  Then:
  $\ent X \ge 0$ and $\cent XY \ge 0$.
\end{fccoro}

\begin{proof}
  Immediate consequence of \cref{thm:1.8}.
\end{proof}

\begin{fccoro}[]
  \label{coro:1.10}
  Assuming:
  - $Y = f(X)$
  Then:
  $\ent Y \le \ent X$.
\end{fccoro}

\begin{proof}
  $\ent X = \ent{X, Y} = \ent Y + \cent XY$. But
  $\cent XY \ge 0$.
\end{proof}

\begin{fcprop}[Subadditivity]
  \label{prop:1.11}
  \label{subadd}
  Assuming:
  - $X$ and $Y$ be random variables
  Then:
  $\ent{X, Y} \le \ent X + \ent Y$.
\end{fcprop}

\begin{proof}
  Note that for any two random variables $X, Y$
  we have
  \begin{align*}
    \ent{X, Y}
    &\le \ent X + \ent Y \\
    \iff
    \cent XY
    &\le \ent X \\
    \iff
    \cent YX
    &\le \ent Y
  \end{align*}
  Next, observe that $\cent XY \le \ent X$ if $X$
  is uniform on a finite set. That is because
  \begin{align*}
    \cent XY
    &= \sum_y \Pbb[Y = y] \cent X{Y = y} \\
    &\le \sum_y \Pbb[Y = y] \ent X
    &&\text{(by \gls{maximality})} \\
    &= \ent X
  \end{align*}
  By the equivalence noted above, we also have
  that $\cent XY \le \ent X$ if $Y$ is uniform.

  Now let $p_{ab} = \Pbb[(X, Y) = (a, b)]$ and
  assume that all $p_{ab}$ are rational. Pick $n$
  such that we can write $p_{ab} =
  \frac{m_{ab}}{n}$ with each $m_{ab}$ an integer.
  Partition $[n]$ into sets $E_{ab}$ of size
  $m_{ab}$. Let $Z$ be uniform on $[n]$. Without
  loss of generality (by \gls{invariance}) $(X,
  Y) = (a, b) \iff Z \in E_{ab}$.

  Let $E_b = \cup_a E_{ab}$ for each $b$. So $Y =
  b \iff Z \in E_b$. Now define a random variable
  $W$ as follows: If $Y = b$, then $W \in E_b$,
  but then $W$ is uniformly distributed in $E_b$
  and independent of $X$ (or $Z$ if you prefer).

  So $W$ and $X$ are conditionally independent
  given $Y$, and $W$ is uniform on $[n]$.

  Then
  \begin{align*}
    \cent XY
    &= \cent X{Y, W}
    &&\text{(by conditional independence)} \\
    &= \cent XW
    &&\text{(as $W$ determines $Y$)} \\
    &\le \ent X
    &&\text{(as $W$ is uniform)}
  \end{align*}
  By \gls{continuity}, we get the result for
  general probabilities.
\end{proof}

\begin{fccoro}[]
  \label{coro:1.12}
  Assuming:
  - $X$ a random variable
  Then:
  $\ent X \ge 0$.
\end{fccoro}

\begin{proof}[Proof (Without using formula)]
  By \nameref{prop:1.11}, $\cent XX \le \ent X$.
  But $\cent XX = 0$.
\end{proof}

\begin{fccoro}[]
  \label{coro:1.13}
  Assuming:
  - $X_1, \ldots, X_n$ are random variables
  Then:
  \[
    \ent{X_1, \ldots, X_n}
    \le \ent{X_1} + \cdots + \ent{X_n}
  .\]
\end{fccoro}

\begin{proof}
  Induction using \nameref{prop:1.11}.
\end{proof}

\begin{fcprop}[Submodularity]
  \label{prop:1.14}
  \label{submod}
  Assuming:
  - $X, Y, Z$ are random variables
  Then:
  \[
    \cent X{Y, Z} \le \cent XZ
  .\]
\end{fcprop}

\begin{proof}
  Calculate:
  \begin{align*}
    \cent X{Y, Z}
    &= \sum_z \Pbb[Z = z] \cent X{Y, Z = z} \\
    &\le \sum_z \Pbb[Z = z] \cent X{Z = z} \\
    &= \cent XZ \qedhere
  \end{align*}
\end{proof}

\nameref{prop:1.14} can be expressed in many ways.

Expanding using \gls{additivity} gives the
following inequalities:
\begin{align*}
  \ent{X, Y, Z} - \ent{Y, Z}
  &\le \ent{X, Z} - \ent Z \\
  \ent{X, Y, Z}
  &\le \ent{X, Z} + \ent{Y, Z} - \ent Z \\
  \ent{X, Y, Z} + \ent Z
  &\le \ent{X, Z} + \ent{Y, Z}
\end{align*}