\documentclass[11pt]{article}
\usepackage{amsmath,amssymb,amsthm,url,dsfont,graphicx,algorithm}
\usepackage[noend]{algpseudocode}

\DeclareMathOperator*{\E}{\mathbb{E}}
\let\Pr\relax
\DeclareMathOperator*{\Pr}{\mathbb{P}}

\newcommand{\eps}{\epsilon}
\newcommand{\inprod}[1]{\left\langle #1 \right\rangle}
\newcommand{\R}{\mathbb{R}}
\newcommand{\T}{\intercal}
\renewcommand{\algorithmicrequire}{\textbf{Input:\ }}
\renewcommand{\algorithmicensure}{\textbf{Output:\ }}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {\bf CS 388R: Randomized Algorithms } \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[4]{\handout{#1}{#2}{#3}{Scribe: #4}{Lecture #1}}

\newtheorem{theorem}{Theorem}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{observation}[theorem]{Observation}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{assumption}[theorem]{Assumption}

% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex

\usepackage{filecontents}

\begin{filecontents}{lec21.bib}
  @MISC{Vershynin10introductionto,
    author = {Roman Vershynin},
    title = {Introduction to the non-asymptotic analysis of random
    matrices},
    year = {2010}
  }

  @book{motwani1995randomized,
    title={Randomized Algorithms},
    author={Motwani, Rajeev and Raghavan, Prabhakar},
    year={1995},
    publisher={Cambridge University Press}
  }

  @article{DBLP:journals/corr/abs-0808-4134,
    author    = {Daniel A. Spielman and
    Shang{-}Hua Teng},
    title     = {Spectral Sparsification of Graphs},
    journal   = {CoRR},
    volume    = {abs/0808.4134},
    year      = {2008},
    url       = {http://arxiv.org/abs/0808.4134},
    timestamp = {Mon, 05 Dec 2011 18:05:37 +0100},
    biburl    = {http://dblp.uni-trier.de/rec/bib/journals/corr/abs-0808-4134},
    bibsource = {dblp computer science bibliography, http://dblp.org}
  } 

  @MISC{WolframMathworldFosters,
    author = {Weisstein, Eric W.},
    title = {Foster's Theorems},
    howpublished = {\url{http://mathworld.wolfram.com/FostersTheorems.html}},
  }

\end{filecontents}

\begin{document}

\lecture{21 --- Nov 18, 2015}{Fall 2015}{Prof.\ Eric Price}{Chad Voegele, 
  Arun Sai}

\section{Overview}
In the last class, we defined the terms \textit{cut sparsifier} and
\textit{spectral sparsifier} and introduced Roman Vershynin (RV) Lemma which
will help us analyse Laplacians of random graphs.  In this lecture, we show how
to produce spectral sparsifiers with $O(n\log{n}/\epsilon^2)$ edges where $n$ is
the number of nodes in the graph and $\epsilon$ is a measure of the quality of
the sparsifier.

\section{Background}
\begin{definition}
For $A$ and $B$ symmetric matrices, $A \preceq B$ 
if $~\forall x,$ $x^\T A x \leq x^\T B x$
\end{definition}

\begin{definition}
\textbf{Graph Laplacian}\\
The Laplacian matrix of a weighted graph 
$G = (V, E, w)$, where $w_{(u,v)}$ is the weight of edge $(u,v)$ is defined by
$$
L_G(u,v) = 
\begin{cases}
  -w_{(u,v)} & \text{if } u \ne v\\
  \sum_z w_{(u,z)} & \text{if } u = v\\
\end{cases}
$$
\end{definition}

The Laplacian can be expressed in terms of differences of standard basis
vectors.
\begin{align*}
  L_G &= \sum_{e = (u,v)} w_e (e_u - e_v) (e_u - e_v)^\T\\
      &= \sum_{e = (u,v)} w_e u_e u_e^\T
\end{align*}
where $e_i$ is the standard basis vector such that $(e_i)_j = \delta_{ij}$ and
for edge $e = (u,v)$, $u_e = e_u - e_v$.

\begin{definition}
\label{specSparsifier}
\textbf{Spectral Sparsifier}\\
A graph $H = (V, E', w')$ is an
$\epsilon-$spectral approximation of a graph $G = (V, E, w)$ if
$$
(1-\epsilon) L_G \preceq L_H \preceq (1 + \epsilon) L_G
$$
where $L_G$, $L_H$ are the Laplacians of graphs $G, H$ respectively.
\end{definition}

Note that $x^\T L_G x = \sum_{e=(u,v)} w_e (x_u - x_v)^2$ is shift invariant. So
in the analysis below, we restrict ourselves to $x$ such that 
$x^\T \mathbf{1} = 0$.

\begin{lemma}
\label{rvlemma}
\textbf{Roman Vershynin Lemma}\\ 
Let $\{X_i\}_{i = 1}^{m}$ be i.i.d random
vectors in $\R^n$, such that each $X_i$ is uniformly bounded
$$
\|X_i\|_2 \leq \kappa, \quad \|\mathbb{E}[X_iX_i^\T]\| \leq 1 \quad \forall i \in [m]
$$
Then
$$
\mathbb{E}\left[\|\frac{1}{m}\sum_{i = 1}^{m}
X_i X_i^\T - \mathbb{E}[XX^\T]\|\right] \leq \kappa 
\sqrt{\frac{\log{n}}{m}}
$$
\end{lemma} 

Last class, we proposed the following randomized algorithm for computing a
spectral sparsifier.

\begin{algorithm}
\label{sprfierAlg}
  \caption{Generates spectral sparsifier}
  \begin{algorithmic}[1]
    \Statex
    \algorithmicrequire $G = (V, E, w)$.
    \algorithmicensure $H = (V, E', w')$, a spectral sparsifier of G
    \For{$m$ times}
       \State Choose each edge $e \in E$ with some probability $p_e$
       \State Add edge $e$ to $E'$ with $w'(e) = \frac{w_e}{m p_e}$
    \EndFor
  \end{algorithmic}
\end{algorithm}

In expectation, the Laplacian of the graph $H$ output by the above algorithm is equal to the Laplacian
of $G$. Let $Y_e = \sqrt{\frac{w_e}{p_e}}u_e$ and let $\{Z_i\}_{i = 1}^{m}$ be independent random variables where $Z_i = Y_e$ with
probability $p_e$. Note that $L_H = \frac{1}{m} \sum_{i=1}^m Z_i Z_i^\T$.
\begin{align*}
  \E[L_H] &= \E\left[\frac{1}{m} \sum_{i=1}^m Z_i Z_i^\T\right] \\
  		  & = \E[Z_1 Z_1^\T]\\
  		  &= \sum_{e \in E} p_e Y_e Y_e^\T\\
          &= \sum_{e \in E} w_e u_e u_e^\T\\
          &= L_G
\end{align*}

In the next section we discuss how to choose $p_e$, a probability distribution
over edges in $G$, that gives us a good spectral sparsifier.
\section{Spectral Sparsifiers}
We start with the simple case of complete graphs, which have a spherical
Laplacian, and move to non-complete graphs in Section \ref{noncompG}. To keep
the analysis simple we only consider unweighted graphs.
\subsection{Complete Graphs}
\label{completeG}
When $G$ is a complete graph, the Laplacian $L_G$ is given by:
$$
L_G = 
\left(\begin{array}{cccc}
  n-1 & -1 & \cdots & -1 \\
  -1 & n-1 & \cdots & -1 \\
  \vdots & \vdots & \ddots & \vdots \\ 
  -1 & -1  & \cdots & n-1 \\
\end{array}
\right)
= n I - \mathbf{1} \mathbf{1}^\T
$$

where $\mathbf{1}$ is a vector of all 1's.

From Definition \ref{specSparsifier}, for $H$ to be a spectral sparsifier, we need that
\begin{align*}
\label{e:complete_cond}
  &(1-\epsilon) L_G \preceq L_H \preceq (1 + \epsilon) L_G \\
\Leftrightarrow ~ &(1-\eps) x^\T L_G x \leq x^\T L_H x \leq (1+\eps) x^\T L_G x 
\qquad \forall x \text{ s.t. } x^\T\mathbf{1} = 0\\
 \Leftrightarrow ~ & | x^\T (L_H - L_G) x| \leq \eps x^\T L_G x \qquad \forall x
  \text{ s.t. } x^\T\mathbf{1} = 0\\
 \Leftrightarrow ~ & ||L_H - L_G||_2 \leq \eps n
\end{align*}
where the last step follows from the assumption that $\mathbf{1}^\T x = 0$ and
$x^\T L_G x =  x^\T\left(n I - \mathbf{1} \mathbf{1}^\T\right)x = n ||x||^2$.

We now show that when $m \geq (\frac{n \log n}{\eps^2})$ and $p_e$ is uniform
over edges, Algorithm \ref{sprfierAlg} outputs an $\epsilon-$spectral
approximatior of $G$.  We have:
$$
p_e = \frac{1}{\binom{n}{2}} = \Theta \left(\frac{1}{n^2}\right)
$$
It is easy to see that the random variables $\{Z_i\}_{i = 1}^{m}$ are uniformly bounded:
\begin{align*}
  \|Z_i\|_2 &= \|Y_e\| \quad \text{(for some edge e)}\\
          &= \sqrt{\frac{w_e}{p_e}} \sqrt{u_e^\T u_e} \\ 
          &= \sqrt{\frac{1}{p_e}} \sqrt{2} \\
          &= \Theta(n)
\end{align*}

Also, from before we have that
$$
\| E\left[ Z_i Z_i^\T \right] \| = \|L_G\| = n
$$

Applying RV Lemma on random variables $X_i = \frac{Z_i}{\sqrt{n}}$, we get
$$
  E\left(\frac{1}{n}\| L_H - L_G \|\right) 
    \leq \sqrt{\frac{n \log n}{m}} \\
$$

Thus
\begin{align*}
  E\left(\| L_H - L_G \|\right) 
    &\leq n \sqrt{\frac{n \log n}{m}} \\
    &\leq \eps n \qquad \mbox{if~} m \geq \left(\frac{n \log n}{\eps^2}\right)
\end{align*}

So for $m \geq (\frac{n \log n}{\eps^2})$, we get an $\epsilon-$approximate
spectral sparsifier of $G$.
\subsection{Non-complete Graphs}
\label{noncompG}
For non-complete graphs, there are two issues that we need to deal with.

\begin{enumerate}
  \item The Laplacian of a non-complete graph need not be spherical. So in order
    to apply RV lemma, rather than looking at random variables $\{Z_i\}_{i =
    1}^{m}$, we look at transformed random variables $\{AZ_i\}_{i = 1}^{m}$ for
    some matrix $A$.
  \item Need to find a better sampling distribution, $p_e$. For example, in the
    case of a barbell graph, we need to return the middle edge to get a good
    sparsifier.
\end{enumerate}

Consider an unweighted graph $G$. Let $U \in \R^{|E| \times n}$ be matrix
representing $|E|$ edges where
\begin{align*}
  U = \left(
    \begin{array}{ccc}
      \rule{1.5em}{0.4pt} & u_1^\T & \rule{1.5em}{0.4pt} \\
      \rule{1.5em}{0.4pt} & u_2^\T & \rule{1.5em}{0.4pt} \\
      & \cdots & \\
      \rule{1.5em}{0.4pt} & u_{|E|}^\T & \rule{1.5em}{0.4pt} \\
    \end{array}
  \right)
\end{align*}

Then the Laplacian for $G$ can be represented as
$$
L_G = U^\T U
$$

$L_G$ is symmetric since $L_G^\T = L_G$. Also we have that $\forall x$, 
$x^\T L_G x = x^\T U^\T U x = \| Ux \| \geq 0$ and thus $L_G$ is
positive semi-definite.
This implies that all eigenvalues of $L_G$ are non-negative. Using the
eigenvalue decomposition of $L_G$, we can express $L_G$ as
$$
L_G = \sum_{i=1}^n \lambda_i b_i b_i^\T
$$
where $\{b_i\}_{i=1}^n$ are orthonormal eigenvectors and $\lambda_i
\geq 0$.

Positive powers of $L_G$ can be calculated by
$$
L_G^{~p} = \sum_{i=1}^n \lambda_i^p b_i b_i^\T
$$

The Moore-Penrose pseudo-inverse of $L_G$ is given by
$$
L_G^\dagger = \sum_{\substack{i=1 \\ \lambda_i \ne 0}}^n 
\lambda_i^{-1} b_i b_i^\T
$$

and thus
$$
(L_G^{^\dagger})^{\frac{1}{2}} = \sum_{\substack{i=1 \\ \lambda_i \ne 0}}^n \lambda_i^{-1/2} b_i b_i^\T
$$

Using these, we can express the projector onto the span of $L_G$ as
\begin{align*}
  \Pi_{L_G} &= \sum_{\substack{i=1 \\ \lambda_i \ne 0}} b_i b_i^\T = (L_G^{\dagger})^{{\frac{1}{2}^\T}} L_G^{^\frac{1}{2}} = L_G^{^\frac{1}{2}} (L_G^{\dagger})^{{\frac{1}{2}^\T}} 
\end{align*}

Note that $\Pi_{L_G}^\T \Pi_{L_G} = \Pi_{L_G}$.

For $H$ to be a spectral sparsifier of $G$, we need that
\begin{align*}
& \ x^\T L_H x = (1 \pm \eps) x^\T L_G x, \quad \forall x\\
\Leftrightarrow & \ x^\T L_H x = (1 \pm \eps) x^\T L_G x,\quad \forall x \text{ s.t. } x^T\mathbf{1} = 0\\
\Leftrightarrow & \ x^\T L_H x = (1 \pm \eps) x^\T L_G x, \quad \forall x \in \text{span}(L_G)
\end{align*}
where the last statement holds when G is a connected graph (because $L_G$ has rank $n-1$ for a connected graph and $L_G\mathbf{1} = 0$). Thus for any $x \in \text{span}(L_G)$:
\begin{align*}
  x &= \Pi_{L_G} x =  (L_G^{\dagger})^{{\frac{1}{2}^\T}} L_G^{~\frac{1}{2}} x = (L_G^{\dagger})^{{\frac{1}{2}^\T}} y
\end{align*}
where $y = L_G^{~\frac{1}{2}} x$. Then the condition becomes:
\begin{align*}
  x^\T L_H x &= (1 \pm \eps) x^\T L_G x, \quad \forall x \in \text{span}(L_G)\\
  \Leftrightarrow~ y^\T L_G^{\dagger^\frac{1}{2}} L_H L_G^{\dagger^{\frac{1}{2}^\T}} y 
  &= (1\pm \eps) y^\T L_G^{\dagger^\frac{1}{2}} L_G L_G^{\dagger^{\frac{1}{2}^\T}} y \\
  &= (1\pm \eps) y^\T L_G^{\dagger^\frac{1}{2}} L_G^{\frac{1}{2}^\T} L_G^\frac{1}{2}  L_G^{\dagger^{\frac{1}{2}^\T}} y \\
  &= (1\pm \eps) y^\T \Pi_{L_G}^\T \Pi_{L_G} y \\
  &= (1\pm \eps) y^\T \Pi_{L_G} y, \quad \forall y
\end{align*}

Subtracting $y^\T \Pi_{L_G} y$ from both sides, we get
\begin{gather*}
  |y^\T (L_G^{\dagger^\frac{1}{2}} L_H L_G^{\dagger^{\frac{1}{2}^\T}} - \Pi_{L_G}) y| 
  \leq \eps y^\T \Pi_{L_G} y = \eps y^\T y \\
  \Leftrightarrow
  \| L_G^{\dagger^\frac{1}{2}} L_H L_G^{\dagger^{\frac{1}{2}^\T}} - \Pi_{L_G} \| \leq \eps
\end{gather*}
We now apply RV Lemma on random variables $A_i = L_G^{\dagger^\frac{1}{2}} Z_i$. Let $\kappa = \max\|{A_i}\|$ and we have:
\begin{align*}
  \|E[A_i A_i^\T]\| &= 
    \|L_G^{\dagger^\frac{1}{2}} E[Z_i Z_i^\T] L_G^{\dagger^{\frac{1}{2}^\T}}\|\\
  &= \|L_G^{\dagger^\frac{1}{2}} L_G L_G^{\dagger^{\frac{1}{2}^\T}}\|\\
  &= \|\Pi_{L_G}\|\\
  &\leq 1
\end{align*}
Applying RV Lemma we get:
\[
 \| \frac{1}{m} \sum A_i A_i^\T - E[A_i A_i^\T] \| = \| L_G^{\dagger^\frac{1}{2}} L_H L_G^{\dagger^{\frac{1}{2}^\T}} - \Pi_{L_G} \|
\leq \kappa \sqrt{\frac{\log n}{m}}
\]

So if $m \geq (\kappa^2 \log{n}/\eps^2)$, we get a $\epsilon-$approximate sparsifier. Note that we haven't yet defined the probability distribution $p_e$. $\kappa$ will depend on the choice of $p_e$. 

To pick a good probability distribution $p_e$ and to compute $\kappa$, we appeal to physical intution. Consider the graph to
represent nodes on a circuit and let $x \in \R^n$ denote the voltages on each node.
The current flow along edge $e$, denoted by $I_e$, from $u$ to $v$ is related to
the voltage drop.  Thus $I_e = x_u - x_v = u_e x$. The flow along all edges
is given by $I = Ux$ where $I \in \R^{|E|}$.

Given a battery on the circuit, with $x_s$ and $x_t$ fixed at some
voltages, we can calculate the rest of the internal voltages $x_v$ using Kirchoff's Laws:
$$
\text{current into vertex} - \text{current out of vertex}
 = \text{external flow}
$$
We know that the external flow is: $
(\Delta \text{ at } s, -\Delta \text{ at } t, 0 \text{ elsewhere })$

For node $v$,
\begin{align*}
  (I_{ext})_v &= \sum_{e=(u,v)} I_e - \sum_{e=(v,u)} I_e\\
              &= \sum_e I_e (U_e)_v\\
              &= (U^\T I)_v\\
              &= (U^\T U x)_v\\
              &= (L_G x)_v
\end{align*}

We know the external flow and want voltages so we compute $x = L_G^{\dagger} I_{ext}$ where
$L_G^{\dagger}$ is the pseudoinverse.

If we set $I_{ext}$ to $u_e$ for some edge $(u,v)$ to indicate that 1 unit of
current is pushed from $u$ to $v$, then $L_G^{\dagger} u_e$ is a vector of all voltages
in the circuit and thus $u_e^\T L_G^\dagger u_e$ is the voltage drop from $u$ to $v$.

From Ohm's Law, we know that $V = I R_{eff}$ where $R_{eff}$ is the effective
resistance. Since we have 1 unit of current, we conclude that 
$R_{eff} = u_e^\T L_G^\dagger u_e $.

We use this fact in our calculation of $\|A_i\|$.
\begin{align*}
  \|A_i\|^2 &= A_i^\T A_i \\
          &= Z_i^\T L_G^{\dagger^{\frac{1}{2}^\T}} L_G^{\dagger^\frac{1}{2}} Z_i \\
          &= Z_i^\T L_G^\dagger Z_i\\
          &= \frac{1}{p_e} u_e^\T L_G^\dagger u_e \\
          &= \frac{R_{eff}}{p_e}
\end{align*}

This suggests to set $p_e \propto R_{eff}$ and after normalizing
$$
p_e = \frac{R_{eff}(e)}{\sum R_{eff}(e)}
$$

Thus $\kappa^2 = \sum R_{eff}(e)$ and we need $m \geq (\left(\sum
R_{eff}(e)\right) \log{n}/\eps^2)$ to get a $\epsilon-$approximate sparsifier.
And finally to compute $\sum R_{eff}(e)$, we use Foster's Theorem.

\begin{theorem}
\label{t:fosters_theorem}
\textbf{Foster's Theorem}\\
Let $R_{eff}(e)$ denote the effective resistance along edge $e$ on a
connected graph of $n$ nodes. Then 
$$
\displaystyle \sum_{e \in E} R_{eff}(e) = n-1
$$
\end{theorem} 

\begin{proof}
Define $P = U L_G^{\dagger} U^\T$. Then
$$
P^2 = U L_G^{\dagger} L_G L_G^{\dagger} U^\T = P
$$

Thus $P$ is a projection matrix, and  all its eigenvalues $\lambda_i \in \{0, 1\}$. Since $L_G$ has rank $n-1$ and $P$ has the same rank as $L_G$, $n-1$ eigenvalues of $P$  are equal to 1 and the rest are 0. From the definition of effective resistance we have:
\begin{gather*}
R_{eff}(e) = u_e^\T L_G^{\dagger} u_e = P_{e,e} \\
\Rightarrow \sum R_{eff}(e) = tr(P) = \sum \lambda_i = n - 1
\end{gather*}
\end{proof}

Finally, we conclude that we need
$m \geq (n \log{n}/\epsilon^2)$ and complete the proof for
non-complete graphs.

\nocite{*}
\bibliographystyle{abbrv}
\bibliography{lec21}

\end{document}
