% vim: tw=50
% 19/11/2022 11AM

\begin{hiddenflashcard}[derivative-is-unique]
Proof that derivative is unique? \\
\cloze{
Write:
\[ \|\alpha(h) - \beta(h)\| = \left\|
\frac{\alpha(\lambda h) + \beta(\lambda
h)}{\lambda} \right\| \]
and consider the limit as $\lambda \to 0$.
}
\end{hiddenflashcard}

\begin{proof}
    Let $h \in \RR^n$, $h \neq 0$. Then
    \[ \alpha(h) - \beta(h) = (\eta(h) - \eps(h))
    \|h\| \]
    Then for $\lambda \in \RR$, $\lambda \neq 0$,
    \begin{align*}
        \|\alpha(h) - \beta(h)\|
        &= \left\| \frac{\alpha(\lambda h) -
        \beta(\lambda h)}{\lambda} \right\| \\
        &= \left\| \frac{(\eta(\lambda h) -
        \eps(\lambda h)) \|\lambda h\|}{\lambda}
        \right\| \\
        &= \|\eta(\lambda h) - \eps(\lambda h)\|
        \|h\| \\
        &\to 0
    \end{align*}
    as $\lambda \to 0$. Hence $\alpha(h) -
    \beta(h)$. Hence $\alpha = \beta$.
\end{proof}

\subsubsection*{Remarks}

\begin{enumerate}[(1)]
    \item To consider differentiability of $f$ at
        $a$, only matters what happens on some
        neighbourhood of $a$ .So definition works
        if instead of $f : \RR^n \to \RR^m$ we
        have $f : \mathcal{N} \to \RR^m$ where
        $\mathcal{N} \subset \RR^m$ is a
        neighbourhood of $a$ or, in particular, if
        $f : B_\delta(a) \to \RR^m$ where $\delta
        > 0$. (Imagine $f$ defined as anything on
        rest of $\RR^n$ and makes no difference).
    \item We can define the $l_1$ and $l_\infty$
        norms by
        \[ \|x\|_1 = d_1(0,x) = \sum_{i = 1}^n
        |x_i| \]
        and $\|x\|_\infty = d_\infty(0, x) =
        \max_i |x_i|$. Note $\|x\|_1 \ge 0$ with
        equality if and only if $x = 0$,
        \[ \|\lambda x\|_1 = |\lambda| \|x\|_1;
        \quad \|x + y\|_1 \le \|x\|_1 + \|y\|_1 \]
        Similarly for $\| \bullet \|_\infty$.
        We've seen that for all $x \in \RR^n$:
        \[ \|x\|_\infty \le \|x\| \le
        \sqrt{n}\|x\|_\infty \]
        and
        \[ \|x\|_\infty \le \|x\|_1 \le
        n\|x\|_\infty \]
        So we can replace $\|\bullet\|$ in the
        definition of derivative by
        $\|\bullet\|_1$ or $\|\bullet\|_\infty$
        and the definition doesn't change.
        Sometimes this is useful for computation.
\end{enumerate}

\noindent
Consider the vector space $\mathcal{L}(\RR^n,
\RR^m)$ of linear maps $\RR^n \to \RR^m$. We have
$\mathcal{L}(\RR^n, \RR^m) \sim \RR^{mn}$ with the
obvious isomorphism (write map as a matrix with
respect to standard bases of $\RR^n$ and $\RR^m$).
\\
So could think about Euclidean norm of a linear
map. But seems a bit unnatural.

\begin{flashcard}[OperatorNorm]
\begin{definition*}
    The \emph{operator norm} on
    $\mathcal{L}(\RR^n, \RR^m)$ is defined by
    \[ \|\alpha\| = \cloze{\sup\{\|\alpha x\| \mid \|x\|
    = 1\}} \]
    \fcscrap{$\alpha x \in \RR^m$ so $\|\bullet\|$ is
    normal Euclidean norm.}
\end{definition*}
\end{flashcard}

\begin{proposition}
    Let $\|\bullet\|$ be the operator norm on $V =
    \mathcal{L}(\RR^n, \RR^m)$. Let $\alpha, \beta
    \in V$. Then
    \begin{enumerate}[(i)]
        \item $\|\alpha\| \ge 0$ with equality if
            and only if $\alpha = 0$;
        \item $\forall \lambda \in \RR$,
            $\|\lambda \alpha\| =
            |\lambda|\|\alpha\|$;
        \item $\|\alpha + \beta\| \le \|\alpha\| +
            \|\beta\|$;
        \item $\forall x \in \RR^n$, $\|\alpha x\|
            \le \|\alpha\| \|x\|$;
        \item $\|\alpha \beta\| \le \|\alpha\|
            \|\beta\|$;
        \item If $\|\bullet\|'$ is the Euclidean
            norm on $V \sim \RR^{mn}$ with the
            standard isomorphism then there are
            constants $c, d > 0$ (depending on $m, n$
            but independent of $\alpha$)
            such that
            \[ c\|\alpha\| \le \|\alpha\|' \le
            d\|\alpha\| \]
    \end{enumerate}
\end{proposition}

\subsubsection*{Remarks}

\begin{enumerate}[(1)]
    \item A linear map from $\RR^n \to \RR^m$ is
        continuous and $\{x \in \RR^n \mid \|x\| =
        1\}$ is compact so the operator norm is
        well-defined.
    \item Notation: standard is $\|\bullet\|$
        refers to operator norm if applied to a
        linear map and Euclidean norm if applied
        to a point of $\RR^n$, unless otherwise
        stated.
\end{enumerate}

\begin{proof}
    \begin{enumerate}[(i) to (iii)]
        \item[(i) to (iii)] Exercise.
        \item[(iv)] Let $x \in \RR^n$. If $x = 0$
            then done. Otherwise, $\alpha(x) =
            \|x\| \alpha \left( \frac{x}{\|x\|}
            \right)$ with $\left\| \frac{x}{\|x\|}
            \right\| = 1$. So
            \[ \|\alpha(x)\| \le \|x\| \|\alpha\| \]
        \item[(v)] Let $x \in \RR^n$ with $\|x\| =
            1$. Then
            \[ \|\alpha \beta x\| \le \|\alpha\|
            \|\beta x\| \le \|\alpha\| \|\beta\|
            \|x\| = \|\alpha\| \|\beta\| \]
            (by (iv) twice). So $\|\alpha \beta\|
            \le \|\alpha\| \|\beta\|$.
        \item[(vi)] Let $x \in \RR^n$ with $\|x\| = 1$.
            \[ \|\alpha x\| \le \sqrt{m} \max_{1
            \le i \le m} |(\alpha x)_i| \]
            Let $A$ be the matrix of $\alpha$ with
            respect to the standard bases $e_1,
            \dots, e_n$ of $\RR^n$ and $f_1,
            \dots, f_m$ of $\RR^n$. Then
            \begin{align*}
                \|\alpha x\|
                &\le \sqrt{m} \max_{1 \le i \le m}
                \left| \sum_{j = 1}^n A_{ij} x_j
                \right| \\
                &\le \sqrt{m} \max_{1 \le i \le m}
                \sum_{j = 1}^n |A_{ij}||x_j| \\
                &\le \sqrt{m} \max_{1 \le i \le n}
                \sum_{j = 1}^n \|\alpha\|' \\
                &= n\sqrt{m} \|\alpha\|'
            \end{align*}
            Hence
            \[ \|\alpha\| \le n\sqrt{m}
            \|\alpha\|' \]
            On the other hand, pick $i, j$ that
            maximise $|A_{ij}|$. Then
            \[ \|\alpha e_j\| \ge \|(\alpha
            e_j)_i\| = |A_{ij}| \]
            But
            \begin{align*}
                \|\alpha\|'
                &\le \sqrt{mn} |A_{ij}| \\
                &\le \sqrt{mn} \|\alpha e_j\| \\
                &\le \sqrt{mn} \|\alpha\|
            \end{align*}
            This proves (vi) with $d = \sqrt{mn}$
            and $c = \frac{1}{n\sqrt{m}}$.
            \qedhere
    \end{enumerate}
\end{proof}

\begin{proposition}
    Let $f : \RR^n \to \RR^m$ which is
    differentiable at $a \in \RR^n$. Then $f$ is
    continuous at $a$.
\end{proposition}

\begin{proof}
    Write
    \[ f(a + h) = f(a) + D f|_a (h) + \eps(h)
    \|h\| \]
    where $\eps(h) \to 0$ as $h \to 0$. Also,
    $\|h\| \to 0$ as $h \to 0$, and $D f|_a$ is
    linear so continuous so $D f|_a(h) \to D
    f|_a(0) = 0$ as $h \to 0$. Thus $f(a + h) \to
    f(a)$ as $h \to 0$.
\end{proof}

\begin{proposition}
    Let $f, g : \RR^n \to \RR^m$ and $\lambda :
    \RR^n \to \RR$ be differentiable at $a \in
    \RR^n$. Then $f + g$, $\lambda f$ are
    differentiable at $a$ with
    \[ D(f + g)|_a = D f|_a + D g|_a \]
    and
    \[ D(\lambda f)|_a (h) = \lambda(a) Df|_a(h) +
    D \lambda|_a (h) f(a) \]
\end{proposition}

\begin{proof}
    We have
    \[ f(a + h) = f(a) + Df|_a(h) + \eps(h) \|h\| \]
    \[ g(a + h) = g(a) + Dg|_a(h) + \eta(h) \|h\| \]
    \[ \lambda(a + h) = \lambda(a) +
    D\lambda|_a(h) + \zeta(h)\|h\| \]
    where $\eps(h), \eta(h), \zeta(h) \to 0$ as $h
    \to 0$. Now,
    \begin{align*}
        (f + g)(a + h)
        &= (f + g)(a) + (Df|_a + Dg|_a)(h) +
        (\eps(h) + \eta(h))\|h\|
    \end{align*}
    where $Df|_a + Dg|_a$ is linear and $\eps(h) +
    \eta(h) \to 0$ as $h \to 0$. \myskip
    Also,
    \begin{align*}
        (\lambda f)(a + h)
        &= (\lambda f)(a) + \lambda(a)
        Df|_a(h) + D\lambda|_a(h) f(a) +
        \zeta(h)\|h\|
    \end{align*}
    where $h \mapsto \lambda(a) Df|_a(h) +
    D\lambda|_a(h) f(a)$ is a linear map, and
    \[ \xi(h) = \zeta(h) f(a) + Df|_a(h)
    D\lambda|_a(h) \frac{1}{\|h\|} +
    Df|_a(h)\zeta(h) + \lambda(a)\eps(h) +
    D\lambda|_a(h) \eps(h) + \eps(h) \zeta(h)\|h\| \]
    which we claim tends to 0 as $h \to 0$ since:
    \begin{itemize}
        \item $\eps(h), \zeta(h) \to 0$ as $h \to
            0$ so $\zeta(h)f(a)$,
            $Df|_a(h)\zeta(h)$,
            $\lambda(a)\eps(h)$,
            $D\lambda|_a(h)\eps(h)$ and
            $\eps(h)\zeta(h) \|h\|$ all tend to 0
            as $h \to 0$.
        \item $Df|_a$, $D\lambda|_a$ are linear so
            continuous, so $Df|_a(h),
            D\lambda|_a(h) \to 0$ as $h \to 0$,
            and
            \begin{align*}
                \|Df|_a(h) D\lambda|_a(h)
                \frac{1}{\|h\|}\|
                &\le
                \|Df|_a(h)\|\|D\lambda|_a(h)\|
                \frac{1}{\|h\|} \\
                &\le \|Df|_a\| \|h\|
                \|D\lambda|_a\|\|h\|
                \frac{1}{\|h\|} \\
                &= \|Df|_a\| \|D\lambda|_a\| \|h\|
                \\
                &\to 0
            \end{align*}
            as $h \to 0$.
    \end{itemize}
    so $\xi(h) \to 0$ as $h \to 0$.
\end{proof}
