\documentclass[11pt]{article}
\usepackage{amsmath,amssymb,amsthm}
\usepackage{mathtools}
\usepackage{xcolor}

\DeclareMathOperator*{\E}{\mathbb{E}}

\DeclarePairedDelimiter\abs{\lvert}{\rvert}
\DeclarePairedDelimiterX{\norm}[1]{\lVert}{\rVert}{#1}

\newcommand{\w}{\omega}
\newcommand{\eps}{\varepsilon}
\newcommand{\T}{\mathsf{T}}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {\bf CS 388R: Randomized Algorithms, Fall 2019 } \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
      \textcolor{red}{\textbf{NOTE:} THESE NOTES HAVE NOT BEEN EDITED OR CHECKED FOR CORRECTNESS}
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[5]{\handout{XX}{#3}{#4}{Scribe: #5}{Lecture #1: #2}}

\newtheorem{theorem}{Theorem}

% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex

\begin{document}

\lecture{21}{Matrix concentration and Graph Sparsification}{November 7th, 2019}{Prof.\ Eric Price}{Ryan Rock, Sabee Grewal}

\section{Overview}

In this lecture we give a proof of the Rudelson-Vershynin (RV) theorem \cite{rudelson_vershynin}, and then begin graph sparsification.

\section{Inequalities}
We derive the Bernstein inequality for scalar random variables, extend this result to symmetric matrices, and then prove the RV theorem.
\subsection{Bernstein Inequality}
Let $X_1, X_2, \hdots, X_n$ be independent random variables such that
\[
  \E\left[\sum_i X_i\right] = 0
  \qquad  \max_i \abs{X_i} \leq K
  \qquad  \sum_i \E\left[X_i^2\right] \leq \sigma^2.
\]
Then $\sum_i X_i = \text{subgamma}(2\sigma^2, 2K)$, which gives the bound
\[
    \Pr\left[\abs*{\sum_i X_i} \geq t\right]
        \leq 2 \exp\left(-\frac{1}{4}\min\left(\frac{t^2}{\sigma^2}, \frac{t}{K}\right)\right).
\]

\subsection{Matrix Bernstein Inequality}
Let $X_1, X_2, \hdots, X_N$ be independent, symmetric matrices in $\mathbb{R}^{n \times n}$ such that
\begin{equation*}
  \E\left[X_i\right] = 0
  \qquad  \max_i \norm{X_i} \leq K
  \qquad  \norm*{\sum_i \E\left[X_i^2\right]} \leq \sigma^2
\end{equation*}
where we define $\norm{}$ as the spectral norm for matrices, the $l^2$ norm for vectors and the absolute value norm for scalars. Analogizing from the scalar case,
\begin{equation}
    \Pr\left[\norm*{\sum_i X_i} \geq t\right]
        \leq 2n \exp\left(-C\min\left(\frac{t^2}{\sigma^2}, \frac{t}{K}\right)\right)
    \label{eq:matrix_bernstein}
\end{equation}
for some constant $C$. For a proof see \cite{tropp}.
\subsection{Rudelson-Vershynin Inequality}
\begin{theorem}
Let there be $m$ independent vectors $X_i \in \mathbb{R}^n$ such that
\[
    \max_i \norm{X_i} \leq K \qquad \norm{\E[X_i X_i^\T]} \leq 1.
\]
Then
\[
    \E\left[\norm*{\frac{1}{m} \sum_i X_i X_i^\T - \frac{1}{m}\sum_i\E[X_i X_i^\T]}\right]
        \lesssim K \sqrt{\frac{\log(n)}{m}},
\]
for $K \sqrt{\frac{\log(n)}{m}} \leq 1$.
\end{theorem}
\begin{proof}
Let $Y_i = X_i X_i^\T - \E[X_i X_i^\T]$. We would like to bound $\norm{\sum_i Y_i}$ similar to equation \eqref{eq:matrix_bernstein}, which can be done by bounding $\max_i \norm{Y_i}$ and $\norm{\sum_i  \E[Y_i^2]}$. We have that
\begin{equation*}
    \begin{split}
        \max_i \norm{Y_i} &\leq \max_i \norm{X_i X_i^\T} + \norm{\E[X_i X_i^\T]}\\
            &\leq \max_i \norm{X_i}^2 + \norm{\E[X_i X_i^\T]}\\
            &\leq K^2 + 1\\
            &\leq 2K^2
    \end{split}
\end{equation*}
and that
\begin{equation*}
    \begin{split}
        \norm*{\sum_i \E[Y_i^2]}
            &\leq \sum_i \norm{\E[(X_i X_i^\T - \E[X_i X_i^\T])^2]}\\
            &\leq \sum_i \norm{\E[X_i X_i^\T X_i X_i^\T] - \E[X_i X_i^\T]^2}\\
            &\leq \sum_i K^2 \norm{\E[X_i X_i^\T]} + \norm{-\E[X_i X_i^\T]^2}\\
            &\leq m (K^2 + 1)\\
            &\leq 2mK^2.
    \end{split}
\end{equation*}
Then using the Matrix Bernstein inequality,
\[
    \Pr\left[\norm*{\sum_i Y_i} \geq t\right]
        \leq 2n \exp\left(-C \min\left(\frac{t^2}{2mK^2}, \frac{t}{2K^2} \right)\right).
\]
for some constant $C$. For our application it's the case that $\frac{t^2}{2mK^2} \leq \frac{t}{2K^2}$, so 
\[
    \E\left[\norm*{\frac{1}{m} \sum_i Y_i}\right] \lesssim K\sqrt{\frac{\log(n)}{m}}.
\]
\end{proof}
For the original proof see theorem 3.1 in \cite{rudelson_vershynin}.
\section{Graph Sparsification}
We approximate a weighted graph $G = (V, E, \w)$ by another graph $H = (V, \tilde{E}, \tilde{\w})$ where $|\tilde{E}| = O(K^2 \log(|V|)/\varepsilon^2)$. We do this by proving through the RV theorem that the Laplacian of $G$ is approximately similar to the Laplacian of $H$, a random matrix that results from sampling edges from $E$ dependent on effective resistances and edge weights. The original material for this section can be seen in \cite{spielman}.
\subsection{Laplacian}
For a weighted graph $G = (V, E, \w)$, we define its Laplacian as
\[
    L_G =
    \begin{cases}
        -\w_{(u,v)}        & \mbox{if } u \neq v\\
        \sum_{z \in \{x | (u, x) \in E\}} w_{(u, z)} & \mbox{if } u = v
    \end{cases}.
\]
This is equivalently defined as
\[
    L_G = \sum_{e \in E} w_e y_e y_e^\T
\]
where $y_e \in \mathbb{Z}^{|V|}$ is the all 0s vector besides $y_e(u) = 1$ and $y_e(v) = -1$.

The Laplacian is useful because we can use it to approximate $G$. For instance, consider $G$ a representation of a circuit and define the total power needed to run the circuit for particular node-voltages as
\[
    P_G(x) = \sum_{e=(u,v)\in E} (x_u - x_v)^2 w_e = x^\T L_G x,
\]
where $x \in \mathbb{R}^{|V|}$ is the node-voltages. For our approximation we would like to prove for all $x$ that
\[
    (1 - \eps)P_G(x) \leq P_H(x) \leq (1 + \eps) P_G(x)
\]
since this implies that
\[
    (1 - \eps) x^\T L_G x \preceq x^\T L_H x \preceq (1 + \eps) x^\T L_G x.
\]
\subsection{Effective Resistance}
In order to determine how likely each edge should be included in an approximation, we analogize edge weights, $w_e$, to conductance, and measure the effective resistances, $R_e$, between two nodes that have an edge. We define $p_e$, the probability of sampling an edge, as
\[
    p_e = w_e R_e.
\]
For an intuitive sense why sampling dependent on $w_e$ and $R_e$ works, note that the effective resistance between two adjacent nodes is the same as the probability that a random spanning tree contains those nodes' connector edge.
\subsection{Spectral Sparsification}
Remember that our goal is to approximate $G$ using a graph $H$ that has only $O(K^2 \log(|V|)/\varepsilon^2)$ edges. We can do this by continually sampling an edge from $G$, modifying this sampled edge's weight and adding this modified edge to $\tilde{H}$, where $\tilde{H}$ starts as an all-0s matrix and tends to $H$ after enough samples. Let $Z$ be a random variable such that
\[
    Z = y_e \sqrt{\frac{w_e}{p_e}} \qquad \text{with probability } p_e.
\]
Then the expectation of $ZZ^\T$ is
\[
    \E\left[Z Z^\T\right] = \sum_{e \in E} p_e \frac{w_e}{p_e} y_e y_e^\T = L_G.
\]

In the next lecture we will show that using the RV theorem and setting $L_H = \frac{1}{m} \sum_{i=1}^m Z_i Z_i^\T$ gives the bound
\[
    \E[\norm{L_H - L_G}] \lesssim K \sqrt{\frac{\log n}{m}}
\]
when $m = O(K^2 \log (|V|)/\eps^2)$.

\bibliographystyle{alpha}
\begin{thebibliography}{42}

\bibitem[RV05]{rudelson_vershynin}
Mark~Rudelson, Roman~Vershynin.
\newblock Sampling from large matrices: an approach through geometric functional analysis.
\newblock {\em arXiv: math/0503442 [math.FA]}, 2005.

\bibitem[SS08]{spielman}
Daniel~Spielman, Nikhil~Srivastava.
\newblock Graph Sparsification by Effective Resistances.
\newblock {\em arXiv: 0803.0929 [cs.DS]}, 2008.

\bibitem[Tro15]{tropp}
Joel~Tropp.
\newblock An Introduction to Matrix Concentration Inequalities.
\newblock {\em arXiv: 1501.01571 [math.PR]}, 2015.

\end{thebibliography}

\end{document}
