\documentclass[11pt]{article}
\usepackage{amsmath,amssymb,amsthm}
\usepackage{mathtools}
\usepackage{xcolor}
\usepackage[bookmarks=true,bookmarksnumbered,bookmarksdepth=2,pdfstartview=FitH,colorlinks,linkcolor=blue,filecolor=blue,citecolor=blue,urlcolor=blue]{hyperref}
\usepackage{enumerate}
\usepackage{mdframed}
\usepackage{blkarray}

\DeclareMathOperator*{\E}{\mathbb{E}}
\let\Pr\relax
\DeclareMathOperator*{\Pr}{\mathbb{P}}
\DeclareMathOperator{\normal}{\mathcal{N}}
\DeclareMathOperator{\Var}{Var}
\DeclareMathOperator{\range}{range}


\DeclarePairedDelimiter{\abs}{\lvert}{\rvert}
\DeclarePairedDelimiter{\norm}{\lVert}{\rVert}

\newcommand{\w}{\omega}
\newcommand{\eps}{\varepsilon}
\newcommand{\T}{\mathsf{T}}

\newcommand*{\vertbar}{\rule[-1ex]{0.5pt}{2.5ex}}
\newcommand*{\horzbar}{\rule[.5ex]{2.5ex}{0.5pt}}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {\bf CS 388R: Randomized Algorithms, Fall 2019 } \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
      \textcolor{red}{\textbf{NOTE:} THESE NOTES HAVE NOT BEEN EDITED OR CHECKED FOR CORRECTNESS}
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[5]{\handout{XX}{#3}{#4}{Scribe: #5}{Lecture #1: #2}}

\newcommand{\namedref}[2]{\hyperref[#2]{#1~\ref*{#2}}}

\newcommand{\sectionref}[1]{\namedref{Section}{sec:#1}}
\newcommand{\subsectionref}[1]{\namedref{Subsection}{subsec:#1}}
\newcommand{\appendixref}[1]{\namedref{Appendix}{app:#1}}
\newcommand{\theoremref}[1]{\namedref{Theorem}{thm:#1}}
\newcommand{\corollaryref}[1]{\namedref{Corollary}{cor:#1}}
\newcommand{\propositionref}[1]{\namedref{Proposition}{prpsn:#1}}
\newcommand{\propertyref}[1]{\namedref{Property}{prop:#1}}
\newcommand{\conjectureref}[1]{\namedref{Conjecture}{conj:#1}}
\newcommand{\assumptionref}[1]{\namedref{Assumption}{assm:#1}}
\newcommand{\definitionref}[1]{\namedref{Definition}{def:#1}}
\newcommand{\lemmaref}[1]{\namedref{Lemma}{lem:#1}}
\newcommand{\claimref}[1]{\namedref{Claim}{clm:#1}}
\newcommand{\tableref}[1]{\namedref{Table}{tab:#1}}
\newcommand{\figureref}[1]{\namedref{Figure}{fig:#1}}
\newcommand{\equationref}[1]{\namedref{Equation}{eq:#1}}
\newcommand{\footnoteref}[1]{\namedref{Footnote}{foot:#1}}


\newtheorem{theorem}{Theorem}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{observation}[theorem]{Observation}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{assumption}[theorem]{Assumption}

% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex

\begin{document}

\lecture{22}{Spectral Sparsification}{November 12th, 2019}{Prof.\ Eric Price}{Sebastian Oberhoff, Rachit Garg}

\section{Overview}

In the previous lecture we discussed:
\begin{itemize}
\item Bernstein Inequality and Matrix Bernstein Inequality.
\item Rudelson-Vershynin Inequality.
\item An introduction to graph sparsification.
\end{itemize}
In this lecture we look at spectral sparsification for complete graphs using the Rudelson-Vershynin Lemma and then analyze it for all undirected graphs.

\section{Setup}

Last time we saw the following Lemma due to Rudelson and Vershynin \cite{rv_lemma}:

\begin{theorem}[RV Lemma]
Let $x_1, \dots, x_m \in \mathbb{R}^n$ be independent and satisfying
\[
\norm{x_i}_2 \leq K \quad {\scriptstyle (K \geq 1)} \qquad \text{and} \qquad \bigl\lVert \E\bigl[x_i x_i^T\bigr] \bigr\rVert \leq 1\,.
\]
Then
\[
\E\Biggl[\biggl\lVert \tfrac{1}{m} \sum_{i=1}^m x_i x_i^T - \E\bigl[x_i x_i^T\bigr]\biggr\rVert \Biggr] \leq \sigma K \sqrt{\tfrac{\log(n)}{m}}\,.
\]
\end{theorem}

Note: For a matrix $M$ we define
\begin{align*}
\lVert M \rVert & = \text{spectral norm}\\
& = \text{maximum eigenvalue (if positive semidefinite)}\\
& = \sup_{y \neq 0} \frac{y^T M y}{\norm{y}_2^2}\\
\end{align*}

Given a graph $G = (V, E)$ define the diagonal matrix
\[
D = \begin{bmatrix} d_1 \\ & \ddots \\ & & d_n \end{bmatrix}
\]
where the $d_i$ are the degrees, as well as the (weighted) adjacency matrix $A$.

This lets us further define the Laplacian
\[
L_G = D - A = \sum_{e \in E} w_e u_e u_e^T
\]
where $w_e$ is the weight on edge $e$ and $u_e$ is of the form
\[
u_e = \begin{blockarray}{cc} \begin{block}{[c]c} \\ -1 & {\scriptstyle j} \\ \\ 1 & {\scriptstyle i} \\ \end{block}\end{blockarray} \qquad \text{\scriptsize ($e$ goes from vertex $i$ to $j$).}
\]

The goal is now: given a dense $G$, find a sparse $H$ ($m = n\log(n)$) such that $L_H \approx L_G$. That is,
\[
\forall x \in \mathbb{R}^n \colon \underbrace{x^T L_H x}_{P_H} = (1 \pm \epsilon) \underbrace{x^T L_G x}_{P_G} \quad \iff \quad (1 - \epsilon) \preceq L_H \preceq (1 + \epsilon) L_G
\]

($P_H, P_G$ is the power on the graphs $H, G$ respectively.)

\subsection*{Approach}

Pick probabilities $p_e$. Set $z_i \coloneqq \sqrt{\frac{w_e}{p_e}} \cdot u_e$ with probability $p_e$. Then
\[
\E\bigl[z_i z_i^T\bigr] = \sum_e p_e \frac{w_e}{p_e} u_e u_e^T = L_G\,.
\]
So after obtaining $z_i, \dots, z_m$ in this way we can construct
\[
L_H \coloneqq \frac{1}{m} \sum_{i=1}^m z_i z_i^T
\]
obeying $\E[L_H] = L_G$.

Question: for what $m$ do we have $L_H \to L_G$?

\section{Warmup: Complete Graph}

For a complete graph we have
\begin{gather*}
L_G =
\begin{bmatrix}
n-1 & -1 & -1\\
-1 & n-1 & -1 & \dots\\
-1 & -1 & n-1\\
\vdots & & & \ddots\\
\end{bmatrix}
= nI - \mathbf{1}\mathbf{1}^T\\[1em]
P_G(x) = x^T L_G x^T = n \norm{x}_2^2 - (\mathbf{1}^T x)^2 \qquad \forall x \text{ s.t. } \mathbf{1}^T x = 0
\end{gather*}

And what we want to show is
\[
\frac{x^T(L_H - L_G)x}{\underbrace{x^T L_G x}_{n\norm{x}_2^2}} \leq \epsilon \quad \iff \quad \norm{L_G - L_H} \leq n \epsilon\,.
\]

By the RV Lemma we have
\[
\E\bigl[\norm{L_H - L_G}\bigr] \leq \sigma K \sqrt{\tfrac{\log(n)}{m}}
\]
where
\[
K = \max_i \norm{z_i} = \max_e \sqrt{\tfrac{w_e}{p_e} \cdot 2} = \sqrt{n(n-1)} \approx n \qquad \text{and} \qquad \sigma = \norm{L_G} = n\,.
\]
Hence,
\[
\E\bigl[\norm{L_H - L_G} \bigr] \leq n \sqrt{n \tfrac{\log(n)}{m}}\,.
\]
So $n = O\bigl(\frac{n\log(n)}{\epsilon^2}\bigr)$ suffices.

\section{Spectral Sparsification for General Graphs}
In the previous section we discussed spectral sparsification for complete graphs using the RV lemma \cite{rv_lemma}. The main observation here is that in the case for complete graphs the distribution was spherical, i.e.\ each edge had the same weight. But in the case of general graphs there could be some edges which are more important than the others. (Imagine the case of a  barbell graph for an intuition. Here the connecting edge should be preserved in order for the two parts of the graph to remain connected). Thus we must appropriately handle the skewed distribution of a non-complete graph. We first introduce some notation and consider the simplifying assumption that the graph is an \textbf{unweighted undirected graph}.

Define $U \in \mathbb{R}^{\abs{E} \times n}$ to be the matrix below, where $\abs{E}$ is the number of edges and $n$ is the number of vertices of the graph (we define $u_e$ for edge $e = (a, b)$ to be $(e_a - e_b)$ where $e_x$ is the elementary vector $\in \{0,1\}^n$):
\begin{gather*}
U = 
\begin{bmatrix}
\horzbar & u_1^T & \horzbar\\
\horzbar & u_2^T & \horzbar\\
\horzbar & \vdots & \horzbar\\
\horzbar & u_m^T & \horzbar\\
\end{bmatrix}\\[1em]
L_G = U^T U,\ L_H = U^TSU
\end{gather*}

where $S \in \mathbb R^{\abs{E}\times\abs{E}}$ is a diagonal matrix such that $S_{e,e} = \frac{\# \text{times $e$ sampled}}{m\,p_e}$ and $m$ is the number of samples.

\subsection{Intuition from Electrical Engineering}

Let $v$ be the voltage at each vertex $\in \mathbb R^n$ and $y$ be the currents across each edge $\in \mathbb R^{\abs{E}}$.

\textbf{Ohm's Law}: states that $V=IR$ or $I = VC$, where $C$ is the conductance. As we consider an unweighted graph, the conductance is $1$ throughout. Hence we get that $I=V$ which says that the potential difference is equal to the current. This implies that $\forall i,j \colon y_{i,j} = v_i-v_j \implies y=VU$.

\textbf{Kirchhoff's Current Law}: Consider injecting current, $x\in \mathbb{R}^n$ into the graph. By Kirchhoff's current law, $x^T \mathbf{1} = 0$, due to charge conservation, sum of all incoming and outgoing currents is zero. Also observe that $x=U^T y = U^TU v = L_G v$. Since every matrix with real entries must have a pseudoinverse, we can write that $v=L_G^+ x$. The use of the voltage vector will be in describing the exact probabilities with which we should choose each edge and sparsify the graph.
%(The pseudo inverse is needed because we need to perform the computation of the inverse orthagonal to the all $1$ vector.) 

Let $r_e$ be the effective resitance of an edge, i.e. it is the potential difference induced when we inject one ampere of current between two junctions.

\begin{align*}
V=IR \implies r_e &=\text{voltage gap when we force $1A$ current across an edge $e$} \\
&= u_e^T v \tag{\footnotesize when $x=u_e$} \\
&= u_e^T L_G^+ u_e
\end{align*}


More generally, if we write the effective resistance $R$ as a matrix where the diagonal entries are the effective resistances, then
\[
R = U L_G^+ U^T
\]
where $r_e = R_{e,e}$.
\begin{claim}
\cite{SS08} If $p_e \propto r_e (p_e = \frac{r_e}{n-1})$, then $m = O(\frac{n\log n}{\epsilon^2})$ suffices for $L_H$ to be spectral sparsifier i.e. $L_H$ approximates $L_G$ with high probability.
\end{claim}

We now look at some properties of $R$

\begin{lemma}R is a projection matrix \label{lem:projR}\end{lemma}
\begin{lemma}$R$ has $n-1$ eigenvalues = $1$, (rest = $0$)\label{lem:eigR} \end{lemma}
\begin{lemma}$\E\bigl[\norm{RSR-R}\bigr] \leq \epsilon$\label{lem:RV-R}\end{lemma}
\begin{lemma}$(1-\epsilon)L_G \preceq L_H \preceq (1+\epsilon)L_G$\label{lem:approxR}\end{lemma}

\lemmaref{projR}
\begin{proof}
\begin{align*}
R^2 & = U L_G^+ (U^T U) L_G^+ U^T\\
& = U (L_G^+ L_G) L_G^+ U^T\\
& = U L_G^+ U^T\\
& = R\,. & & \qedhere\\
\end{align*}
\end{proof}

\newpage

\lemmaref{eigR}
\begin{proof}
This can be seen easily due to $R$ being a projection matrix. Let $v$ be an eigenvector and $\lambda$ be the corresponding eigenvalue i.e. $Rv = \lambda v$. $R^2v = R\lambda v = \lambda^2 v$. But $R^2$ and $R$ are the same matrices, hence $\lambda^2=\lambda \implies \lambda \in \{0,1\}$.

\begin{theorem}[Foster's theorem]
$R$ has $n-1$ eigenvalues = 1 (rest 0).
\end{theorem}
Using Foster's theorem we also get that $\sum_e r_e = \operatorname{Tr}(R) = n-1$.
\end{proof}

\lemmaref{approxR}
\begin{proof}
Assume that \lemmaref{RV-R} is true for now. Using Markov's we have the fact that $\norm{RSR-R} \leq c\epsilon$ with good probability for some constant $c$. Note that we can achieve high probability over here if we revisit the proof of \cite{rv_lemma}. Ignoring the constant $c$ as we perform an asymptotic analysis at the end.

We have
\begin{align*}
\epsilon & \geq \norm{RSR-R}\\
& = \norm{R(S-I)R}\\
& =\sup_{y\neq 0} \frac{y^TR(S-I)Ry}{y^Ty}\,.
\end{align*}

We want to bound
\begin{align*}
\sup_{x}\frac{x^T(L_H-L_G)x}{x^TL_Gx} & = \sup_{x}\frac{x^TU^T(S-I)U x}{x^TU^TU x}\\
& = \sup_{y = U x}\frac{y^T(S-I)y}{y^Ty}\\
\intertext{\footnotesize \centering (Note that $y = U x = U L_G^+ U U^T x = R U^T x$. Thus, $y$ lies in the range of $R$ always.)}
& = \sup_{y \in \range(R),~y=Ry'}\frac{y'^TR^T(S-I)Ry'}{y'^TR^TRy'}\\
& = \sup_{y \in \range(R),~y=Ry'}\frac{y'^TR(S-I)Ry'}{y'^TRRy'}\\
& = \sup_{y \in \range(R),~y=Ry'}\frac{y'^TR(S-I)Ry'}{y'^Ty'} \tag{\footnotesize as $\norm{R} = 1$}\\
& \leq \epsilon\,.
\end{align*}
This gives us that
\[
(1-\epsilon)L_G \preceq L_H \preceq (1+\epsilon)L_G\,. \qedhere
\]
\end{proof}

\newpage

\lemmaref{RV-R}
\begin{proof}
\begin{align*}
RSR & = \sum_e S_{e,e}\cdot R_e R_e^T \tag{\footnotesize where $R_e \in \mathbb R^n$ is the $e^{th}$ column of $R$}\\
& = \sum_e \frac{\# \text{ times e sampled}}{m\,p_e} R_e R_e^T\,. & & \qedhere\\ 
\end{align*}
\end{proof}

Set $z_i = \frac{R_e}{\sqrt p_e}$ with probability $p_e$, then $S = \sum_{i=1}^m z_i z_i^T$, has $\E[S] = R$.

$RV$ lemma uses
\[
\norm{\E[z_i z_i^T]} = \norm{R} = 1
\]
\begin{equation}
\norm{z_i} = \frac{1}{\sqrt{p_e}}\norm{R_e}_2 = \frac{\sqrt r_e}{\sqrt p_e} = \sqrt {n-1} \label{eq:nexttime}
\end{equation}

and from the statement of the lemma, we get that
\[
\E\bigl[\norm{RSR-R}\bigr] \leq \sqrt{\frac{(n-1)\log n}{m}} \leq \epsilon\,.
\]

\eqref{eq:nexttime} uses the fact that $\|R_e\|_2 = \sqrt{r_e}$ which will see in the next class to complete the proof.

\bibliographystyle{alpha}
\begin{thebibliography}{42}

\bibitem[RV05]{rv_lemma}
Mark~Rudelson, Roman~Vershynin.
\newblock Sampling from large matrices: an approach through geometric functional analysis.
\newblock {\em arXiv: math/0503442 [math.FA]}, 2005.

\bibitem[SS08]{SS08}
Daniel~Spielman, Nikhil~Srivastava.
\newblock Graph Sparsification by Effective Resistances.
\newblock {\em arXiv: 0803.0929 [cs.DS]}, 2008.

\iffalse
\bibitem[Tro15]{tropp}
Joel~Tropp.
\newblock An Introduction to Matrix Concentration Inequalities.
\newblock {\em arXiv: 1501.01571 [math.PR]}, 2015.
\fi

\end{thebibliography}

\end{document}
