% vim: tw=50
% 23/11/2022 11AM

\subsection{Gram Schmidt and orthogonal complement}

\begin{itemize}
    \item $V$ vector space over $\RR$ (or $\CC$).
        An inner product is a \emph{positive
        definite} symmetric (or Hermitian)
        bilinear form on $V$.
        \begin{notation*}
            $\varphi(u, v) = \langle u, v
            \rangle$.
        \end{notation*}
    \item Norm: $\|v\| = \sqrt{\langle v, v
        \rangle}$ (the norm). Then $\|v\| \ge 0$
        and $\|v\| = 0 \iff v = 0$.
\end{itemize}
$\to$ associated notion of length.

\begin{lemma*}[Cauchy-Schwartz]
    \[ |\langle v, v \rangle| \le \|u\| \|v\| \]
    More over, equality holds if and only if $u$
    and $v$ are proportional.
\end{lemma*}

\begin{proof}
    $f = \RR$ or $\CC$. Let $t \in F$, then
    \begin{align*}
        0
        &\le \|tu - v\|^2 \\
        &= \langle tu - v, tu - v \rangle \\
        &= t\ol{t} \langle u, u \rangle - t
        \langle v, v \rangle - \ol{t} \langle v, u
        \rangle + \|v\|^2 \\
        &= |t|^2 \|u\|^2 - 2 \Re(t\langle v, u
        \rangle) + \|v\|^2
    \end{align*}
    Explicitly: the minimum is taken at $t =
    \frac{\ol{\langle u, v \rangle}}{\|u\|^2}$
    \[ \implies 0 \le \frac{|\langle u, v
    \rangle|^2}{\|u\|^2} \|u\|^2 - 2\Re \left(
    \frac{|\langle u, v \rangle|^2}{\|u\|^2}
    \right) + \|v\|^2 \]
    \[ \implies 0 \le \|v\|^2 - \frac{|\langle u,
    v \rangle|^2}{\|u\|^2} \]
    \[ \implies |\langle u, v \rangle|^2 \le
    \|u\|^2 \|v\|^2 \]
    Exercise: equality $\implies$ $u$ and $v$ are
    proportional.
\end{proof}

\begin{corollary*}[Triangle inequality]
    \[ \|u + v\| \le \|u\| + \|v\| \tag{$*$} \]
\end{corollary*}

\noindent
$\to$ key to show that $\|\bullet\|$ is a norm.

\begin{proof}
    \begin{align*}
        \|u + v\|^2
        &= \langle u + v, u + v \rangle \\
        &= \|u\|^2 + 2\Re(\langle u, v \rangle) +
        \|v\|^2 \\
        &\le \|u\|^2 + 2\|u\| \|v\| + \|v\|^2 \\
        &= (\|u\| + \|v\|)^2 \qedhere
    \end{align*}
\end{proof}

\begin{definition*}
    A set $(e_1, \dots, e_k)$ of vectors of $V$ is
    \begin{enumerate}[(i)]
        \item Orthogonal: if $\langle e_i, e_j
            \rangle = 0$ if $i \neq j$.
        \item Orthonormal: if $\langle e_i, e_i
            \rangle = \delta_{ij}$ where
            \[ \delta_{ij} = \begin{cases}
                1 & i = j \\
                0 & i \neq j
            \end{cases} \]
    \end{enumerate}
\end{definition*}

\begin{lemma*}
    If $(e_1, \dots, e_k)$ is orthogonal, then
    \begin{enumerate}[(i)]
        \item The family is free
        \item $v = \sum_{j = 1}^k \lambda_j e_j$,
            then
            \[ \lambda_j = \frac{\langle v, e_j
            \rangle}{\|e_j\|^2} \]
    \end{enumerate}
\end{lemma*}

\begin{proof}
    \begin{enumerate}[(i)]
        \item $\sum_{j = 1}^k \lambda_j e_j = 0$
            \[ \implies 0 = \left\langle \sum_{j =
            1}^k \lambda_j e_j, e_i \right\rangle
            = \sum_{j = 1}^k \lambda_j \langle
            e_j, e_i \rangle = \lambda_i \]
            so the family is free.
        \item $v = \sum_{i = 1}^k \lambda_i e_i$.
            \[ \implies \langle v, e_j \rangle =
            \lambda_j \langle e_j, e_j \rangle =
            \lambda \|e_j\|^2 \]
            \[ \implies \lambda_j =
            \frac{1}{\|e_j\|^2} \langle v, e_j
            \rangle \qedhere \]
    \end{enumerate}
\end{proof}

\begin{lemma*}[Parseval's Identity]
    If $V$ is a finite dimensional inner product
    space and $(e_1, \dots, e_n)$ is an
    \emph{orthonormal} basis, then
    \[ \langle u, v \rangle = \sum_{i = 1}^n
    \langle u, e_i \rangle \ol{\langle v, e_i
    \rangle} \]
    In particular, in an \emph{orthonormal basis},
    \[ \|v\|^2 = \langle v, v \rangle = \sum_{i =
    1}^n |\langle v, e_i \rangle|^2 \]
    \[ v = \sum_{i = 1}^n \langle v, e_i \rangle
    e_i \]
    ($\|e_i\| = 1$)
\end{lemma*}

\begin{proof}
    $u = \sum_{i = 1}^n \langle e, e_i \rangle
    e_i$, $\|e_1\| = 1$, $v = \sum_{i = 1}^n
    \langle v, e_i \rangle e_i$
    \[ \implies \langle u, v \rangle \left\langle
    \sum_{i = 1}^n \langle u, e_i \rangle e_i,
    \sum_{i = 1}^n \langle v, e_i \rangle e_i
    \right\rangle = \sum_{i = 1}^n \langle u, e_i
    \rangle \ol{\langle v, e_i \rangle} \qedhere \]
\end{proof}

\begin{theorem*}[Gram-Schmidt orthogonalisation
    process]
    $V$ inner product space let $I$ countable
    (finite) est and $(v_i)_{i \in I}$ linearly
    independent. Then there exists a sequence
    $(e_i)_{i \in I}$ of \emph{orthonormal}
    vectors such that
    \[ \Span \langle v_1, \dots, v_k \rangle =
    \Span \langle e_1, \dots, e_k \rangle \]
    $\forall k \ge 1$.
\end{theorem*}

$\to$ if $\dim V < \infty$, then we have existence
of an \emph{orthonormal} basis.

\begin{proof}
    We construct the $(e_i)_{i \in I}$ family by
    induction on $k$.
    \begin{itemize}
        \item $k = 1$, $v_1 \neq 0 \implies e_1 =
            \frac{v_1}{\|v_1\|}$.
        \item Say we found $(e_1, \dots, e_k)$ ,
            we look for $e_{k + 1}$.
            \begin{center}
                \includegraphics[width=0.6\linewidth]
                {images/0ee772386b2311ed.png}
            \end{center}
            We define:
            \[ e_{k + 1}' = v_{k + 1} - \sum_{i =
            1}^k \langle v_{k + 1}, e_i \rangle
            e_i \]
        \item $e_{k + 1}' \neq 0$. Indeed,
            otherwise,
            \[ v_{k + 1} \in \langle e_1, \dots,
            e_k \rangle = \langle v_1, \dots, v_k
            \rangle \]
            which would contradict the fact that
            $(v_i)_{i \in I}$ is free.
        \item Pick $1 \le j \le k$:
            \begin{align*}
                \langle e_{k + 1}', e_j \rangle
                &= \left\langle v_{k + 1} -
                \sum_{i = 1}^k \langle v_{k +
                1}, e_i \rangle e_i, e_j
                \right\rangle \\
                &= \langle v_{k + 1}, e_j \rangle
                - \langle v_{k + 1}, e_j \rangle
                \\
                &= 0
            \end{align*}
        \item $\langle v_1, \dots, v_{k + 1}
            \rangle = \langle e_1, \dots, e_k,
            e_{k + 1}' \rangle$.
        \item We take $e_{k + 1} = \frac{e_{k +
            1}'}{\|e_{k + !}'\|}$ \qedhere
    \end{itemize}
\end{proof}

\myskip
$\implies$ Gram Schmidt designs an algorithm to
compute $e_k$ for all $k$.

\begin{corollary*}
    $V$ \emph{finite dimensional} inner product
    space. Then any orthonormal set of vectors can
    be extended to an orthonormal basis of $V$.
\end{corollary*}

\begin{proof}
    Pick $(e_1, \dots, e_k)$ orthonormal. Then
    they are linearly independent, so we can
    extend $(e_1, \dots, e_k, v_{k + 1}, \dots,
    v_n)$ basis of $V$. Apply Gram-Schmidt to this
    set noticing that there is no need to modify
    the first $k$ vectors.
    \[ \implies (e_1, \dots, e_k, e_{k + 1},
    \dots, e_n) \]
    orthonormal basis of $V$.
\end{proof}

\begin{note*}
    $A \in \mathcal{M}_n(\RR)$, then $A$ has
    \emph{orthonormal} column vectors if and only
    if
    \[ A^\top A = \id \qquad (\RR) \]
    \[ A^\top \ol{A} = \id \qquad (\CC) \]
\end{note*}

\begin{definition*}
    \begin{enumerate}[(i)]
        \item $A \in \mathcal{M}_n(\RR)$ is
            \emph{orthogonal} if:
            \[ A^\top A = \id \qquad (\iff A^{-1} =
            A^\top) \]
        \item $A \in \mathcal{M}_n(\CC)$ is
            unitary if:
            \[ A^\top \ol{A} = \id \qquad (\iff
            A^{-1} = \ol{A^\top}) \]
    \end{enumerate}
\end{definition*}

\begin{proposition*}
    $A \in \mathcal{M}_n(\RR)$ (respectively
    $\mathcal{M}_n(\CC)$), then $A$ can be written
    $A = RT$ where:
    \begin{itemize}
        \item $T$ is upper triangular
        \item $R$ is orthogonal (respectively
            unitary)
    \end{itemize}
\end{proposition*}

\begin{proof}
    Exercise: apply Gram Schmidt to the $(c_1,
    \dots, c_n)$ column vectors of $A$.
\end{proof}

\subsubsection*{Orthogonal complement and
projection}

\begin{definition*}
    \begin{itemize}
        \item $V$ inner product space
        \item $V_1, V_2 \le V$.
    \end{itemize}
    We say that $V$ is the \emph{orthogonal}
    direct sum of $V_1$ and $V_2$ if:
    \begin{enumerate}[(i)]
        \item $V = V_1 \oplus V_2$
        \item $\forall v_1, v_2) \in V_1 \times
            V_2$, $\langle v_1, v_2 \rangle = 0$
    \end{enumerate}
    \begin{notation*}
        $V = V_1 \orthoplus V_2$ ($V =
        V_1 + V_2$) TODO\dots
    \end{notation*}
\end{definition*}

\begin{remark*}
    $v \in V_1 \cap V_2$, $\|v\|^2 = \langle v, v
    \rangle = 0 \implies v = 0$.
\end{remark*}

\begin{definition*}[orthogonal]
    $V$ inner product space, $W \le V$.
    \[ W^\perp = \{v \in V \mid \langle v, w
    \rangle = 0 ~ \forall w \in W\} =
    \text{orthogonal of $W$} \]
\end{definition*}

\begin{lemma*}
    $V$ inner product space, $\dim V < \infty$, $W
    \le V$. Then
    \[ V = W \orthoplus W^\perp
    \tag{$*$} \]
\end{lemma*}