\documentclass[11pt]{article}
\usepackage{amsmath,amssymb,amsthm}

\DeclareMathOperator*{\E}{\mathbb{E}}
\let\Pr\relax
\DeclareMathOperator*{\Pr}{\mathbb{P}}

\newcommand{\eps}{\epsilon}
\newcommand{\inprod}[1]{\left\langle #1 \right\rangle}
\newcommand{\R}{\mathbb{R}}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {\bf CS 395T: Sublinear Algorithms } \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[4]{\handout{#1}{#2}{#3}{Scribe: #4}{Lecture #1}}

\newtheorem{theorem}{Theorem}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{observation}[theorem]{Observation}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{assumption}[theorem]{Assumption}

% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex

\begin{document}

\lecture{13 --- October 6, 2016}{Fall 2016}{Prof.\ Eric Price}
{Kiyeon Jeon and Loc Hoang}

\section{Overview}

In the last lecture we covered the lower bound for $p^{\text{th}}$ moment 
$(p>2)$ and the concepts of packing numbers, covering numbers, and metric entropy.

In this lecture we discuss \textbf{Maurey's Empirical Method} for covering numbers
and begin moving into compressed sensing by starting with \textbf{Restricted
Isometric Property (RIP) matrices}.

\section{Covering Numbers and Maurey's Empirical Method}

\subsection{Introduction}

Last lecture, we discussed the problem of getting a covering number $N$ for
$L_1$ balls using $L_2$ balls.

\begin{equation}
N (\epsilon, B_1^d, ||\cdot||_2)
\end{equation}

Using a volume argument, we were able to establish the following result.
\begin{align}
N (\epsilon, B_1^d, ||\cdot||_2) &\leq  N (\epsilon, B_1^d, ||\cdot||_1)\\
N (\epsilon, B_1^d, ||\cdot||_1) &{}\leq (1 + \frac{2}{\epsilon})^d
\end{align}

The first inequality comes from the fact that $L_1$ balls take up less
space than $L_2$ balls, so it would take more of them to do the covering.

From this, we could get metric entropy $\log{N}$.

\begin{equation}
\log{N} \geq d \log{\frac{1}{\epsilon}} - \frac{d}{2} \log{d}
\end{equation}

We can then deduce that $\log{N} = \theta(d \log{\frac{1}{\epsilon}})$ if
$\epsilon < d^{-\frac{1}{2} - \Omega(1)}$. We are interested in the
case where $\epsilon > \frac{1}{\sqrt{d}}$, and this will be examined
in the next section.


\subsection{Using Maurey's Empirical Method for Covering Numbers}

Using Maurey's empirical method, we will show the following:

\begin{theorem}
When $\epsilon > \frac{1}{\sqrt{d}}$, $N \leq (2d + 1)^{O(1 / \epsilon^2)}$ 

As a result, $\log{N} \lesssim \frac{1}{\epsilon^2} \log(d)$.
\end{theorem}

\begin{proof}
Let's cover the following set:
\begin{align*}
B_1^{d,+}=\{x\in\mathcal{R}^d\,|\,\|x\|_1\leq 1 \text{ and } x_i\geq 0 \,\, \forall 
i \}
\end{align*}

The above set means that $\sum x_i \leq 1 \,\, \forall x_i\geq 0$. 

We can think about a probability distribution over 
$\{e_1, \ldots, e_d, 0\}$:
\begin{align*}
z=\sum_{i=1}^d x_i e_i + (1-\|x\|_1)\cdot 0
\end{align*}

This implies the following probabilities.
\begin{align*}
\Pr&[z=e_j]=x_j \, \forall j \in [d]\\
\Pr&[z=0]=1-\|x\|_1
\end{align*}

With these, we can get a mean of the probability distribution.
\begin{align*}
\E[z]
&=\sum \Pr[z=e_j]\cdot e_j +\Pr[z=0]\cdot 0=\sum x_j\cdot e_j=x
\end{align*}

We will draw $t$ samples $z_1, \ldots, z_t$ from the distribution where
each $z$ is some $e_i$. After drawing the samples, we can take the average
of the samples:
\begin{align*}
\bar{z}=\frac{1}{t}\sum_{i=1}^t z_i
\end{align*}

We want to show that $\E[\|\bar{z}-x\|_2^2] \leq \eps^2$. If we can
do this, then if we take all possible $\bar{z}$, we get an 
$\eps$-cover of the space using those $\bar{z}$ since then all $x$
we can choose will be within $\eps$ of some point in the cover by what
we argue above.

We work this out below:
\begin{align*}
\E[\|\bar{z}-x\|_2^2] &= \E[\sum_{j=1}^d (\bar{z_j}-x_j)^2]\\
&=\sum_{j=1}^d \text{Var }(\bar{z_j}) \text{(definition of variance)}\\
&=\sum_{j=1}^d \frac{1}{t^2} \text{Var } (\sum_{i=1}^{t} (z_i)_j)
\text{(independent variances means you can sum them)}\\
&=\frac{1}{t}\sum_{j=1}^d \text{Var }((z_1)_j)\\
&=\frac{\sum_{j=1}^d x_j}{t} = \frac{\|x\|_1}{t}\\
&\leq \frac{1}{t} \text{(by our original definition/choice of $x$)}
\end{align*}

Now, let $t = 1/\eps^2$. We have the following desired bound:
\begin{align*}
\E[\|\bar{z}-x\|_2^2]\leq\eps^2
\end{align*}

This implies that there exists 
$\bar{z}$ with $\|\bar{z}-x\|_2\leq \eps$ if $t = 1 / \eps^2$.
Then we can pick all $\bar{z}$ to create our $\eps$-cover. 
The number of such
$\bar{z}$ is $\leq (d+1)^t = (d+1)^{O(1/\eps^2)}$ (sample
$t$ $z$s, and there are $d + 1$ choices for each $z$, then
take the mean).

Therefore, we have a bound on packing number and a bound
on the metric entropy.
\begin{align*}
N &\leq (d+1)^{O(1/\eps^2)}\\
\log{N} &\leq \frac{1}{\eps^2}\log(d+1)
\end{align*}

This implies $\log{N} \lesssim \frac{1}{\eps^2}\log(d)$ as desired.

\end{proof}

Note that this could be extended to cover a larger space (note
we only cover $B_1^{d,+}$, which is a positive space). The basic
proof idea will still go through if we decided to extend it to 
larger cases.

\section{Restricted Isometric Property (RIP) Matrices}

We move the discussion towards RIP matricies, which will move us closer
to compressed sensing.

\subsection{Definition of RIP}
We say a vector $x\in\mathcal{R}^n$ is $k$-sparse if 
$|\text{supp}(x)|=\|x\|_0\leq k$.  


Define 
$T_k:=\{x\in\mathcal{R}^n | \|x\|_0 \leq k, \|x\|_2\leq 1\} \subseteq \mathcal{R}^n$,
or the set of all vectors that are $k$-sparse as well as have an $L_2$ norm that 
is less than 1. We want to determine the a bound on the metric entropy of $T_k$, 
$\log{N} ( \epsilon, T_k, \| \cdot \|_2)$. 
To do so, we look at $\log{N} ( \epsilon, T_n, \| \cdot \|_2)$ ($T_n$ 
ignores sparsity). From
what we covered last class, we can determine that 
$\log{N} ( \epsilon, T_n, \| \cdot \|_2) \leq (1 + \frac{2}{\epsilon} )^d$.
From here, we can take a union bound over $k$-dimensional subspaces (by
using ${n \choose k}$) to bound our original packing number:
\begin{align*}
N(\eps, T_k, \| \cdot \|_2)
&\leq \binom{n}{k}(1+\frac{2}{\eps})^k\\
&\leq(\frac{en}{k})^k(1+\frac{2}{\eps})^k\\
\log N(\eps, T_k, \|\cdot\|_2) &\leq k \log(\frac{n*2e}{\eps k})
\end{align*}

This bound is good as it is something that depends reasonably on $k$.
Using this, we can get a bound on RIP matrices.

\begin{definition}
A matrix $A\in\mathcal{R}^{m\times n}$ is a $(k,\eps)$ RIP (Restricted 
Isometry Property) matrix of order $(k,\eps)$ 
if $\forall k$-sparse $x$, $\|Ax\|_2 = (1\pm\eps)\|x\|_x$
\end{definition}

RIP matrices are useful for recovery of vectors:

\begin{theorem} Let $y = Ax + e$ where $x$ is $k$-sparse and 
$A$ has $(O(k), .1)$-RIP. 
Then, one can recover an $\hat{x}$ such that 
$\|\hat{x}-x\|_2\leq O(\|e\|_2)$ 

Also, let $y = Ax$ where $x$ is not $k$-sparse and $A$ has RIP.  Then, 
$\|\hat{x}-x\|_1\leq O(\|x-x_k\|_1)$
\end{theorem}

We can compare the first bound on $\hat{x}$ to Count-Sketch. In Count-Sketch,
we have $Ax$ where $x$ is not $k$-sparse. Then, w.h.p. we get
get $\hat{x}$ with $\|\hat{x}-x\|_2\leq(1+\eps)\|x-x_k\|_2$. The second
bound of the theorem is comparable to Count-Min.

Basically, RIP lets us observe good results once we have selected a good
$A$ with the RIP.

\subsection{Analysis of Number of Rows ($m$) for a RIP Matrix}

\paragraph{How large does $m$ need to be?}
We need to determine how many rows $m$ there needs to be in a matrix $A$
in order to satisfy $(k,\eps)$ RIP.

Pick $A$ as an i.i.d. (sub)gaussian matrix.  For any fixed $x$, we have that
$\|Ax\|_2^2=(1\pm\eps)\|x\|_2^2$ w.p. $1-\exp^{-\eps^2 m \cdot \Omega(1)}$.

We want to be able to get a bound for \textbf{all} $x$, not fixed $x$. 
We could use a union bound, but the problem is that there are infinitely many
$x$. This is where we can use metric entropy: close vectors $x_i$ (in some 
space) will be similar in behavior. Therefore, it suffices to take a union
bound over an $\epsilon$-cover of $T_k$. 
$m = O(\frac{1}{\eps^2} k \log \frac{n}{\eps k})$ will work. This $m$
is needed so that we can use the Johnson-Lindenstrauss bound on the
$L_2$ norm later in the proof below.

\paragraph{Proof of Existence of an $A$ satisifying $(k,\eps)$ RIP}

Let $C$ be an $\eps$-cover of $T_k$ built as $\binom{n}{k} \eps$-covers 
of subspaces with support size $k$. 
Now, take $x^{\ast}\in T_k$. 
\begin{align*}
x^{\ast}=x_1+x', \|x'\| \leq \eps , x_1\in C
\end{align*}

From the fact that $x_1 \in C$ and because of a special property of our cover
$C$ (the way it was built), we have that 
$|\text{supp}(x_1) \cup \text{supp}(x^{\ast})|\leq k$. 
From this we have that $\|x'\|_0 \leq k$. since we are ``rounding'' within
the same subspace.
Now that $\|x'\|\leq \eps$ (since $x^{\ast}$ is at most $\eps$ far from
$x_1$ since $C$ is an $\eps$-cover) and $\|x'\|_0\leq k$, we can say $x'\in \eps T_k$. 

Similarly to $x^{\ast}$ above, we can find $x_2 \in C$ and 
$\|x''\| \leq \eps$ satyisfying
\begin{align*}
x'=\eps(x_2+x'')
\end{align*}

Again, with a similar argument to the one we made for $x'$ above, 
we can obtain $\|x''\|_0\leq k$. Again, we can find an $x_3 \in C$ and
$\|x'''\| \leq \eps$ like above, and we can do this over and over.

As we do this process continuously, we derive the following:
\begin{align*}
x^{\ast}
&=x_1+\eps x_2+ \eps^2 x_3+\cdots\eps^{l-1}x_{l-1}+x^{(l)}\\
&\text{where } \|x^{(l)}\|\leq\eps^l, \|x^{(l)}\|_0\leq k, x_i\in C \,\,\forall i=1,\ldots, l-1
\end{align*}

Now, let $m = O(\frac{1}{\eps^2} k \log \frac{n}{\eps k})$. 
This size of $m$ allows us to use Johnson-Lindenstrauus for a tail bound, 
specifically $\|Ax\|=(1\pm\eps)\|x\|$ $\forall x \in C$ by a union bound
(see lecture 10 of the Fall 2014 version of the class). We use this below.  
\begin{align*}
\|Ax^{\ast}\|
&\leq\|Ax_1\|+\eps\|Ax_2\|+\eps^2\|Ax_3\|+\cdots\\
&\leq(1+\eps)(1+\eps+\eps^2+\cdots)\\
&\leq\frac{1+\eps}{1-\eps}=1+O(\eps)\\
\|Ax^{\ast}\|
&\geq\|x^{\ast}\|-O(\eps)
\end{align*}


So, $\forall x \in T_k$,
\begin{align*}
&\|x\|-O(\eps)\leq\|Ax\|\leq(1+O(\eps))\\
&\forall x \in T_k \text{ with } \|x\|=1, \|Ax\|=(1+O(\eps))\|x\| \rightarrow\\
&\forall x \in T_k\,\, \|Ax\|=(1+O(\eps))\|x\| \rightarrow\\
&A \text{ satisfies } (k, O(\eps))\text{-RIP } \text{if } m=O(\frac{1}{\eps^2}k \log(\frac{n}{\eps k}))
\end{align*}


\subsection{Further Considerations and Extensions}
\subsubsection{Other Considerations for $A$}

\begin{enumerate}
\item What about small integer entries in $A$?

What if $A_{ij} \in \{\pm\frac{1}{\sqrt{m}}\}$ i.i.d.?

It turns out the proof will work for any subgaussian generated matrix.
We only used the Gaussian generated matrix for the JL property for the tail bound.

\item Can we get $A$ to be sparse so we can do computations faster?

This can't happen: shown on homework.

\item Can we still hope to store $A$ more efficiently with limited independence?

Probably not, but note that $A$ (a matrix with the RIP property) can be a fixed
matrix (i.e. find once, then reuse).

\item Can we get an explicit matrix? 

We can show a $k^{1.9\times}$ construction.

\item Are there other matrices that fast to store and compute?

We look at Fourier matrices in the following section.
\end{enumerate}


\subsubsection{Extensions}

\textbf{Question}: 
How might we use the Fourier matrix to get a RIP matrix that is easier to deal with?

We have $\|Fx\|_2=\|x\|_2$ (note it preserves the $L_2$ norm) 
where $F_{ij}=\frac{1}{\sqrt{n}}\exp^{2\pi \sqrt{-1}\frac{i\cdot j}{n}}$. 
By choosing $\Omega \subseteq [n]$ at random (i.e. choose random rows), 
we get $F_{\Omega} \times \sqrt{\frac{n}{m}}$. 
If $|\Omega|\geq\frac{1}{\eps^2}k\log n \log^2 k$, 
then $F_{\Omega}$ has $(k,\eps)$-RIP with ``good'' probability.

(note that there have been better results for the last term
in the inequality above: $\log^5 k['06], \log^3 k['08], \log^2 k['16]$)

\begin{center}
  \begin{tabular}{| l | c | c |}
    \hline
                                          & Gaussian & Fourier \\ \hline
    space to store                       & $mn$ & $m$ \\ \hline
    time to multiply                     & $mn$ & $n\log n$ \\ \hline
    time to multiply by $k$-sparse vector & $mk$ & $\min(mk,n\log n)$ \\
    \hline
  \end{tabular}
\end{center}

We can also use Circulant matrices. Such a matrix $C_v$ is shown below.

\begin{center}
  \begin{tabular}{ |c  c  c  c  c |}
    \hline
    $v_1$  & $v_2$  &$\cdots$ & $v_{n-1}$ & $v_n$ \\
    $v_2$  & $v_3$  &$\cdots$ & $v_n$     & $v_1$ \\
    $\vdots$  & $\vdots$        &       & $\vdots$ & $\vdots$ \\
    $v_n$  & $v_1$  &$\cdots$ & $v_{n-2}$ & $v_{n-1}$ \\
    \hline
  \end{tabular}
\end{center}

Similar to what we did for Fourier matrices, we choose 
$(C_v)_{\Omega}$ where $\Omega$ is \textbf{arbitrary} on Circulant matrices. 
Unlike Fourier, where row
selection had to be random, we can choose whichever rows we want without the need
for randomess.  $v$, however, is random (e.g. Gaussian).  

The number of rows $m$ is $O(k \log n \log^3(k\log n))$.

\end{document}
