\documentclass[11pt]{article}
\usepackage{amsmath,amssymb,amsthm}
\usepackage{wasysym}
\usepackage{amssymb }

\DeclareMathOperator*{\E}{\mathbb{E}}
\let\Pr\relax
\DeclareMathOperator*{\Pr}{\mathbb{P}}

\newcommand{\eps}{\epsilon}
\newcommand{\inprod}[1]{\left\langle #1 \right\rangle}
\newcommand{\R}{\mathbb{R}}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {\bf CS 388R: Randomized Algorithms } \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[4]{\handout{#1}{#2}{#3}{Scribe: #4}{Lecture #1}}
\newcommand{\?}{\stackrel{?}{\leq}}

\newtheorem{theorem}{Theorem}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{observation}[theorem]{Observation}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{assumption}[theorem]{Assumption}

% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex

\begin{document}

\lecture{19 --- 7 Nov 2015}{Fall 2017}{Prof.\ Eric Price}{Isfar Tariq, Srilakshmi Pattabiraman}

\section{Overview}
In this lecture we will do the non-commutative Bernstein Inequality and Graph Sparsification problem.

\section{Bernstein Inequality}
Let $X_1, X_2, \dots, X_n$ bet $n$ independent, not necessarily identically distributed random variables. Further, 
\begin{align*}
\vert X_i \vert &\leq K \quad \forall i \in [n] \\
\mathbb{E}\left[ \sum_{i = 1}^{n} X_i^2 \right] &\leq \sigma^2.
\end{align*}

We wish to find the tail bounds for $\vert \sum_{i=1}^{n} X_i \vert$, \textit{i.e.}, $\mathbb{P}\left[ \vert \sum_{i=1}^{n} X_i \vert \geq t \right] \leq ?$ \\

Note that $X_i$s are sub-gaussian($K$). This in turn implies that $\sum_{i=1}^{n} X_i$ is sub-gaussian($K\sqrt{n}$). Thus,

\begin{align*}
\mathbb{P}\left[ \left| \sum_{i=1}^{n} X_i \right| \geq t \right] \leq e^{-\frac{t^2}{2K^2n}}.
\end{align*} 

This in turn implies $\left| \sum_{i=1}^{n} X_i \right| \simeq K \sqrt{n}$. However, note that the bound is weak when $\sigma \ll K \sqrt{n}$.

Note that $X_i$s are also sub-gamma random variables. $\mathbb{E}[X_i^2] \leq \sigma_i^2 K^2$, $|X_i| \leq K$ implies that $X_i$ is sub-gamma($2\sqrt{2} \sigma_i K,4K$). Let us assume $\sigma_i$ is such that it subsumes $K$ in the argument. Thus, $X_i \in \text{sub-gamma}(2\sqrt{2} \sigma_i,4K)$, and $\sum _{i =1}^{n} X_i \in \text{sub-gamma}(2\sqrt{2} \sigma,4K)$. Using bounds for sub-gamma random variables, we can now write 
\begin{align*}
\mathbb{P}\left[ \left| \sum_{i=1}^{n} X_i \right| \geq t \right] \leq 2 e^{-\min \left\lbrace \frac{t^2}{16\sigma^2}, \frac{t}{8K} \right\rbrace}.
\end{align*}
But the mean may not be $0$. We use $\mathbb{E}[\sum_{i=1}^n X_i] \leq \mathbb{E}[\sum_{i=1}^n X_i^2]^{\frac{1}{2}} = \sigma$ to write
\begin{align*}
\mathbb{P}\left[ \left| \sum_{i=1}^{n} X_i \right| \geq t \right] & \stackrel{(a)}{\leq} 2 e^{-\min \left\lbrace \frac{(t-\sigma)^2}{16\sigma^2}, \frac{(t-\sigma)}{8K} \right\rbrace} \\
& \leq 2 e^{C{-\min \left\lbrace \frac{t^2}{\sigma^2}, \frac{t}{K} \right\rbrace}},
\end{align*}
where $C>0$ is some constant. Also, note that $(a)$ is meaningful only if $(t-\sigma) \geq 4\sigma \sqrt{\ln(2)}$.
 
\textbf{Notation:} $\|A\| = \sup_{\|x\| \leq 1} \|Ax\|_2$.

\begin{theorem}[Non-commutative Bernstein inequality] Extension of Bernstein-type inequalities to matrices.

Let $X_1, \dots, X_m$ be independent symmetric matrices with zero mean, \textit{i.e.}, $\mathbb{E}[X_i] = 0 \quad \forall i \in [m]$. Also, $\|X_i\| \leq K \forall i \in [m]$, and $\left\|\displaystyle\sum_{i=1}^{n} \mathbb{E}[X_i^2]\right\| \leq \sigma^2$. Then, $\exists \ C < 0$, such that $$\mathbb{P}\left[\left\| \sum_{i=1}^{n} X_i \right\| \geq t\right] \leq 2n \cdot e^{C \min{\left\lbrace \frac{t^2}{\sigma^2}, \frac{t}{K}\right\rbrace }}$$
\end{theorem}

We omit the proof of this theorem.

\begin{theorem}[R-V theorem] 
Let $X_1, \dots, X_m$ be independent, and identically distributed vectors in $\mathbb{R}^{n}$ such that $\|X_i\|_2 \leq K$ ( $K \geq 1$), and $\|\mathbb{E}[X_iX_i^{\top}] \| \leq 1$ $\forall i \in [m]$. Then, $$\mathbb{E}\left[\left\|\frac{1}{m}\sum_{i=1}^{m} X_i X_i^{\top} - \mathbb{E}[XX^{\top}]\right\| \right] \apprle K \sqrt{\frac{\log n}{m}}$$
\end{theorem}

\begin{proof}
Let $Y_i = X_iX_i^{\top} - \mathbb{E}[X_iX_i^{\top}]$. We want to apply the non-commutative Bernstein theorem to $ \sum_{i=1}^{m} Y_i$. 

\underline{Upper bound for $Y$:}
\begin{align*}
\left\| Y_i \right\| \leq \left\| X_iX_i^{\top}\right\| + \left\| \mathbb{E}\left[ X_iX_i^{\top} \right] \right\| \leq 2K^2
\end{align*}

\underline{Upper bound for $\left\| \sum_{i=1}^{m} \mathbb{E}[Y_i^2] \right\| $}
\begin{align*}
	\left\|\displaystyle\sum_{i=1}^{m} \mathbb{E}[Y_i^2]\right\| &\leq m\|\mathbb{E}[Y_1^2]\|\\
		&= m \left\| \mathbb{E}\left[(XX^{\top})^2 - \mathbb{E}[XX^{\top}]^2\right] \right\| \\
		&\leq m\left(\left\|\mathbb{E} [ \|X\|_2^2\cdot XX^{\top}] \right\| + \left\|\mathbb{E}[XX^{\top}]\right\|^2\right) \\
	&\leq 2mK^2
\end{align*}
We can now apply the non-commutative Bernstein inequality.
$$\mathbb{P}\left[\left\|\sum_{i=1}^{m} \mathbb{E}[Y_i]\right\| \geq mt\right] \leq 2n \cdot e^{-C \min{\left(\frac{mt^2}{2K^2}, \frac{mt}{K^2}\right)}}$$

Hence, when $t \geq  \frac{K^2}{m}\log\left( \frac{n}{\delta} \right)$, and $t \geq K \sqrt{\frac{\log(\frac{n}{\delta})}{m}}$

$$\mathbb{P}\left[\left\|\sum_{i=1}^{m} \mathbb{E}[Y_i]\right\| \geq C_2 K \sqrt{\frac{\log(\frac{n}{\delta})}{m}} \right] \leq \delta$$
\end{proof}

More on the subject can be found here \cite{StateArt}.

\section{Graph Sparsifier}

Graph Sparsification problem is the following: Given a dense graph $G = (V, E_G, W_G)$, find a sparse graph $H = (V, E_H, W_H)$, which \textit{approximately} preserves some properties of G. The vertex set will remain the same, but the edge set and their weights can be different. We will henceforth denote $|V|$ by $n$. 

\subsection{Cut-Sparsifier}

In the first lecture we studied a randomized algorithm to compute the min-cut in a graph. Here we study a related problem of finding a \textbf{cut-sparsifier}, namely, a sparse graph $H$, that approximately \textbf{preserves all the cuts} in $G$.

For a given graph $G = (V,E,W)$, a cut $S \subseteq V$ has size:
 $$C_G(S) = \sum_{(u,v) \in E} W(u,v) \cdot \mathbb{I}_{\{ u \in S, v \notin S \}}$$

\begin{definition}[Cut-sparsifier]
$H$ is a cut-sparsifier for $G$ if:
 $$\forall S \subseteq V, C_H(S) = (1 \pm \epsilon )C_G(S)$$
\end{definition}

\subsection{Spectral Sparsifier}

The Spectral Sparsifier is a generalized form of cut-sparsification \cite{SpectralSparse}. Let us define
$$L_G = \sum_{(u,v) \in E_G} A_{u,v}.$$
so that,
\[
L_G(u,v) = \begin{cases}
                    -W(u,v)           & u \neq v \\
                    \sum_{t} W(u,t)   & u = v    \\
                \end{cases}
\]
$L_G$ is called the \textbf{Laplacian Matrix} of the graph. Let $P_G (x)  =  x^{\top} L_G x$

\begin{definition}[Spectral Sparsifier] A spectral sparsifier is a graph that spectrally approximates the graph Laplacian. i.e. for all vectors $x$, we should have

$$P_H(x) = (1 \pm \epsilon) P_G(x)$$
$$\Leftrightarrow  (1 - \epsilon) x^{\top} L_G x  \leq x^{\top} L_H x \leq (1 + \epsilon) x^{\top} L_G x \ \ \ \forall \ x \in \mathbb{R}^n$$
$$\Leftrightarrow  (1 - \epsilon) L_G \preceq L_H  \preceq (1 + \epsilon) L_G $$

\end{definition}

\textbf{Notation:} $\preceq$ is the generalized matrix inequality on symmetric matrices: two symmetric matrices $A$ and $B$ satisfy $A \preceq B$ iff $(B-A)$ is positive semidefinite.


\begin{theorem}
    Spectral Sparsifier $\implies$ Cut-sparsifier
    \end{theorem}
\begin{proof}
Will be done in next class.
%    Let S be any subset of vertices. Let $x = \mathbb{I}_{S}$. Since $(x_u - x_v)^2 = 1$ iff $(u,v)$ has an endpoint in $S$, and the other endpoint out of $S$ (so $(u,v)$ is in the cut defined by $S$), we have $P_G(x) = C_G(x)$
    \end{proof}

%Thus, for cut sparsification, we just need 
%$$ (1 - \epsilon) x^{\top} L_G x  \leq x^{\top} L_H x \leq (1 + \epsilon) x^{\top} L_G x \ \ \ \forall \ x \in \lbrace 0, 1 \rbrace ^ n$$

  \bibliographystyle{abbrv}
  \bibliography{lec18}
\end{document}