\documentclass[11pt]{article}
\usepackage{amsmath,amssymb,amsthm}

\DeclareMathOperator*{\E}{\mathbb{E}}
\let\Pr\relax
\DeclareMathOperator*{\Pr}{\mathbb{P}}

\newcommand{\eps}{\epsilon}
\newcommand{\inprod}[1]{\left\langle #1 \right\rangle}
\newcommand{\R}{\mathbb{R}}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {\bf CS 395T: Sublinear Algorithms } \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[4]{\handout{#1}{#2}{#3}{Scribes: #4}{Lecture #1}}

\newtheorem{theorem}{Theorem}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{observation}[theorem]{Observation}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{assumption}[theorem]{Assumption}

\usepackage{mathtools,xparse}
\DeclarePairedDelimiter{\abs}{\lvert}{\rvert}
\DeclarePairedDelimiter{\norm}{\lVert}{\rVert}
\makeatletter
\let\oldabs\abs
\def\abs{\@ifstar{\oldabs}{\oldabs*}}
%
\let\oldnorm\norm
\def\norm{\@ifstar{\oldnorm}{\oldnorm*}}
\makeatother


% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex

\begin{document}

\lecture{21 --- November 8, 2016}{Fall 2016}{Prof.\ Eric Price}{William Hoza and Vasileios-Orestis Papadigenopoulos}

\section{Overview}

Last time, we began a proof that a matrix consisting of a random small subset of the rows of a Fourier matrix has the RIP ``in expectation.'' Today, we complete the proof of this theorem.

Let $F \in \mathbb{C}^{n \times n}$ be a Fourier matrix. For our purposes, the important properties of this matrix are that for all $i, j$, $|F_{ij}|\leq 1$, and \[\langle F_i, F_j \rangle = \begin{cases} 0 & \text{if } i \neq j \\ n & \text{if } i = j. \end{cases}\] Let $\Omega$ to be a multiset of $m$ random rows of the Fourier matrix $F$. Observe that for any set $s \subseteq [n]$,
\[
	\mathbb{E}[F^{T}_{\Omega \times s} F_{s \times \Omega}] = m \mathbb{E}_{i \in [n]}[F^T_i F_i] = m I_n.
\]
Define
\begin{align*}
\Delta \stackrel{\text{def}}{=} \mathbb{E}\left[\sup_{s : |s| \leq k} \norm{I_{k \times k} - \frac{1}{m} F^{T}_{\Omega \times s} F_{s \times \Omega} }\right],
\end{align*}
so that $\Delta$ is a measure of the expected RIP error of $F_{\Omega}$. Our goal is to show that $\Delta$ is small.

\section{Recap of last class}
First we symmetrize and Gaussianize: to bound $\Delta$, it suffices to bound
\begin{align}
\E_{\Omega, g}\left[\norm{\sum_{i \in \Omega} g_i x^s_i {x^s_i}^T}\right], \label{eqn:1}
\end{align}
where $x_i$ is the column vector $F^T_i$ and $g_1, \dots, g_m$ are independent Gaussians. In fact, we will bound the expectation over $g$ alone in Equation~\ref{eqn:1} for an \emph{arbitrary} $\Omega$. Let $\Sigma_k$ be the set of $k$-sparse unit vectors. Then
\begin{align*}
(\text{\ref{eqn:1}}) \leq \mathbb{E}\left[\sup_{y_i \in \Sigma_k} \left| \sum_{i \in \Omega} g_i \langle x_i,y\rangle ^2\right|\right] \leq 2 \mathbb{E}\left[\sup_{y_i \in \Sigma_k} \underbrace{\sum_{i \in \Omega} g_i \langle x_i,y\rangle^2}_{G_y}\right].
\end{align*}
This is a Gaussian process. Last class, we showed that it can be bounded by Dudley's entropy integral, giving
\begin{align*}
m \Delta \lesssim \int^{\infty}_{0} \sqrt{\log N(\Sigma_k, d, u)} \; du,
\end{align*}
where $d(y,z) = \mathbb{E}[{(G_y - G_z)^2}]^{\frac{1}{2}}$.

\section{Bounding Dudley's entropy integral}

\subsection{Bounding the Gaussian process metric}
By the definition of our Gaussian process,
\begin{align*}
G_y - G_z = \sum_{i \in \Omega} g_i(\langle x_i,y \rangle^2 - \langle x_i,z \rangle^2),
\end{align*}
and hence
\begin{align*}
d(y,z) &= (\sum_i (\langle x_i,y\rangle^2 - \langle x_i, z\rangle^2)^2)^{ \frac{1}{2}} \\
&= (\sum_i((\langle x_i,y+z\rangle \langle x_i,y-z\rangle)^2))^{\frac{1}{2}} \\
&\leq (\sum_i(\langle x_i,y+z\rangle)^2)^{\frac{1}{2}}) \max_{i \in \Omega} |\langle x_i, y-z\rangle |\\
&= \norm{F_{\Omega}(y+z)}_2 \norm{F_\Omega (y-z)}_{\infty} \\
&\leq (2 \sup_{y' \in \Sigma_k} \norm{F_{\Omega} y'}) \norm{F_\Omega (y-z)}_{\infty} \\
&\leq 2 \sqrt{(1+\Delta) m} \norm{F_{\Omega} (y-z)}_{\infty},
\end{align*}
where the last inequality follows from the fact that:
\begin{align*}
\forall y \in \Sigma_k: y^T \left(I - \frac{1}{m} F_{\Omega}^T F_{\Omega}\right) y \leq 1 + \Delta \\
\Rightarrow \norm{F_{\Omega} - y}^2_2 \leq m(1 + \Delta) \Rightarrow \norm{F_{\Omega} - y}_2 \leq \sqrt{m(1 + \Delta)}.
\end{align*}
In general, if $d, d'$ are two metrics on a set $X$ with $d \leq d'$ everywhere, $N(X, d, u) \leq N(X, d', u)$, just by the definition of $N$. Therefore, 
\begin{align*}
m \Delta &\lesssim \int^{+ \infty}_0 \sqrt{\log N(\Sigma_k, d, u)} du \\
&\leq \int_0^\infty \sqrt{\log N(\Sigma_k, \norm{F_\Omega}_\infty, \frac{u}{2 \sqrt{m(1+\Delta)}})} \; du \\
&= 2 \sqrt{m(1+\Delta)} \int^{\infty}_{0} \sqrt{\log N(\Sigma_k, \norm{F_\Omega}_\infty, u)} \; du.
\end{align*}
\subsection{Road map for the rest of the proof}
Our goal, now, is to show something like
\begin{align} \label{eqn:goal}
\int^{\infty}_{0} \sqrt{\log N(\Sigma_k, \norm{F_\Omega \cdot}_\infty, u)} \; du \lesssim \sqrt{k}.
\end{align}
If we manage to do that, we'll be able to conclude that
\begin{align*}
\sqrt{m} \Delta \lesssim \sqrt{1+\Delta} \sqrt{k} \Rightarrow \Delta \lesssim \sqrt{1+\Delta} \sqrt{\frac{k}{m}}.
\end{align*}
This would mean that for some $m \lesssim \frac{1}{\epsilon^2} k$, $\Delta \leq \epsilon \sqrt{1 + \Delta}$, which implies $\Delta \lesssim \epsilon$ like we want. (In fact, we'll prove a slightly weaker bound than Equation~\ref{eqn:goal}, leading to a slightly larger number of measurements $m$.)

\subsection{Bounding $N(\Sigma_k, \|\cdot\|_{F, \infty}, u)$ using the $\|\cdot\|_1$ norm}

Let $\norm{y}_{F, \infty} = \norm{F y}_\infty$. By Cauchy-Schwartz, $\Sigma_k \subseteq B_1 \cdot \sqrt{k}$. Therefore,
\begin{align*}
\int^{\infty}_{0} \sqrt{\log N(\Sigma_k, \norm{F_\Omega \cdot}_\infty, u)} \; du &\leq \int^{\infty}_{0} \sqrt{\log N(\Sigma_k, \|\cdot\|_{F, \infty}, u)} \; du \\
&\leq \sqrt{k}\int^{\infty}_{0} \sqrt{\log N(B_1, \|\cdot\|_{F, \infty}, u)} \; du.
\end{align*}
To bound $N(B_1, \|\cdot\|_{F, \infty}, u)$, observe that since $|F_{ij}| \leq 1$ for every $i, j$, $\|y\|_{F, \infty} \leq \|y\|_1$. Therefore,
\begin{align*}
N(B_1, \norm{\cdot}_{F_\infty}, u) \leq N(B_1, \norm{\cdot }_1, u).
\end{align*}
Using volume ratios, we proved in a previous lecture that 
\begin{align} \label{eqn:l1}
N(B_1, \|\cdot\|_1, u) \leq \left(1 + \frac{2}{u}\right)^n.
\end{align}
Plugging this into our integral, we obtain
\begin{align*}
\int_0^1 \sqrt{\log N} \; du \leq \int_0^1 \sqrt{n} \sqrt{\log(1 + \frac{2}{u})} \lesssim \sqrt{n},
\end{align*}
which is not a very good bound. (Remember, we are shooting for a bound that is more like $\sqrt{k}$.)

\subsection{Bounding $N(\Sigma_k, \|\cdot\|_{F, \infty}, u)$ using Maurey's empirical method}

We are going to apply Maurey's Empirical Method to get a better bound. Choose any $y \in B_1$. Then $y$ belongs to the convex hull of $\{\pm e_i, 0\}$. Randomly round $y$ to some $z_r \in \{\pm e_i, 0\}$ in such a way that $\mathbb{E}[z_r] = y$. We do this for $r = 1 \dots R$, independently each time. There is some sufficiently large $R$ be such that
\begin{align*}
\mathbb{E}\left[\norm{\frac{1}{R} \sum_{r=1}^{\infty} z_r -y}_{F, \infty}\right] \leq u, \forall y.
\end{align*}
For such an $R$, we have $N \leq (2n +1)^R$, which is a much better bound on $N$ as long as $R$ is not too big. So now we turn to bounding $R$. Define $\sigma_R = \mathbb{E}\left[\norm{\frac{1}{R} \sum^R_{i=1}z_i -y}_{F, \infty}\right]$.

To bound $\sigma_R$, we symmetrize. Draw $z_1', z_2', \dots z_R' \sim$ same distribution. Then:
\begin{align*}
\sigma_R &= \mathbb{E}\left[\norm{\frac{1}{R} \sum^R_{i=1}z_i -y}_{F, \infty}\right] \\
&= \mathbb{E}\left[\norm{\frac{1}{R} \sum^R_{r=1}z_r -\mathbb{E}\left[\frac{1}{R} \sum_{r=1}^R z'_r\right]}_{F, \infty}\right] \\
&\leq \mathbb{E}\left[\norm{\frac{1}{R} \sum^R_{r=1} s_r (z_r - z'_r)}\right] \text{\quad(} s_r \in \{\pm 1 \}, \text{i.i.d})\\
&\leq 2 \mathbb{E}\left[\norm{\frac{1}{R} \sum^R_{r=1} s_r z_r}\right]
\end{align*}
For any particular coordinate $i$,
\begin{align*}
\langle \frac{1}{R} \sum^R_{r=1} s_r z_r, x_i \rangle = \frac{1}{R} \sum^R_{r=1} s_r \langle z_r, x_i \rangle
\end{align*}
Notice that since $\langle z_r, x_i \rangle \in \{0,1\}$, this is $\leq 1$ in magnitude. Hence, we can apply the Chernoff bound.
\begin{align*}
\Pr\left[\langle\frac{1}{R} \sum{z_r s_r},x\rangle \geq t\right] \leq e^{-t^2 R \over 2} \\
\Rightarrow \Pr\left[\norm{\frac{1}{R} \sum z_r s_r}_{F, \infty} \geq t\right] \leq 2 n e^{- t^2 R \over 2} \\
\Rightarrow \mathbb{E}\left[\norm{\frac{1}{R} \sum z_r s_r}_{F, \infty}\right] \lesssim \frac{1}{\sqrt{R}} \sqrt{\log n} \text{\quad (} = u \text{ for } (2u+1)^R \text{)}
\end{align*}
Therefore setting $R \leftarrow \frac{\log n}{u^2}$ we get:
\begin{align} \label{eqn:maury}
N(B_1, \norm{\cdot}_{F, \infty}, u) \leq (2n +1)^{\mathcal{O}(\frac{\log n}{u^2})}
\end{align}

\subsection{Combining the bounds}
If we just tried to plug Equation~\ref{eqn:maury} into our integral, we would get a bound of $\int_0^1 \frac{1}{u} \log n \; du$, which diverges. Instead, we plug in the minimum of the two bounds (Equation~\ref{eqn:maury} and Equation~\ref{eqn:l1}):
\begin{align*}
\int^1_0 \min(\frac{1}{u} \log n, \sqrt{n} \sqrt{\log\frac{1}{u}}) du = \int_0^{\frac{1}{n}} \sqrt{n} \sqrt{\log \frac{1}{n}} du + \int^1_{\frac{1}{n}} \frac{1}{u} \log n du
\end{align*}
To conclude:
\begin{align*}
m \Delta \lesssim \sqrt{m(\Delta+1)} \sqrt{k} \log^2 n
\end{align*}
To achieve $\Delta \leq \epsilon$, we choose $m \gtrsim \frac{1}{\epsilon^2} k \log^4 n$.

\subsection{Conclusions}
The proof we just finished is by Rudelson and Vershynin \cite{rv06}. Recently, Haviv and Regev \cite{hr16} improved the bound to $O(\frac{1}{\epsilon^2}k \log n \log^2 k)$. Even this improved bound is worse than the corresponding bound for Gaussian matrices, but Fourier matrices have the benefit of fast multiplication via the FFT.

We've merely bounded the RIP error ``in expectation''. One can show that if the number of measurements $m$ is increased by a factor of $C$, the \emph{probability} that the RIP fails is at most $e^{-C^2}$.

Recall that when we proved that Gaussian matrices have the RIP, we used the fact that they satisfy the JL condition. Krahmer and Ward \cite{kw11} showed that there is a connection going the other direction (RIP $\implies$ JL.) In particular, suppose $A$ satisfies the $(k, \epsilon)$ RIP. We can't hope for $A$ itself to satisfy the JL condition, but let $D$ be a diagonal matrix with i.i.d. uniform $\pm 1$ entries on its diagonal. Krahmer and Ward showed that $AD$ has the $(4\epsilon, e^{-\Omega(k)})$ JL condition.

To get a feel for this bound, recall that a Gaussian matrix with 
\[
	m \lesssim\frac{1}{\epsilon^2} \log \frac{1}{\delta}
\]
rows has the $(\epsilon, \delta)$ JL condition. By our earlier proof, this shows that a Gaussian matrix with
\[
	m \lesssim \frac{1}{\epsilon^2} k \log(n/k)
\]
rows has the $(k, \epsilon)$ RIP. By applying the Krahmer-Ward bound, this implies that a Gaussian matrix with
\[
	m \lesssim \frac{1}{\epsilon^2} \log \frac{1}{\delta} \log \frac{n}{\log(1/\delta)}
\]
rows has the $(\epsilon, \delta)$ JL condition. So we lose something, but not much.

By combining the Krahmer-Ward bound with the proof that Fourier matrices satisfy the RIP, we get a Fourier-based matrix which satisfies the $(\epsilon, \delta)$ JL condition with
\[
	\frac{1}{\epsilon^2} \log \frac{1}{\delta} \log^2 \log \frac{1}{\delta} \log n
\]
rows. Even with the diagonal sign flipping, this matrix still allows for $O(n \log n)$ embedding.

Naturally, this raises the question of whether we can get the ``best of both worlds'' for JL embeddings. That is, can we construct a JL matrix with $O(\frac{1}{\epsilon^2}\log(/\delta))$ rows which admits $O(n \log n)$ time embedding? In general, this is an open question, but if
\[
	m < \frac{\sqrt{n}}{\sqrt{\log n} \log \log \frac{1}{\delta}},
\]
then yes, by simply composing a Fourier embedding with a Gaussian embedding.

\bibliographystyle{alpha}
\begin{thebibliography}{KW11}
	
	\bibitem[HR16]{hr16}
	Ishay Haviv and Oded Regev.
	\newblock The restricted isometry property of subsampled Fourier matrices.
	\newblock In {\em Proceedings of the Twenty-Seventh Annual ACM-SIAM Symposium
		on Discrete Algorithms}, pages 288--297. SIAM, 2016.
	
	\bibitem[KW11]{kw11}
	Felix Krahmer and Rachel Ward.
	\newblock New and improved Johnson-Lindenstrauss embeddings via the restricted
	isometry property.
	\newblock {\em SIAM Journal on Mathematical Analysis}, 43(3):1269--1281, 2011.
	
	\bibitem[RV06]{rv06}
	Mark Rudelson and Roman Vershynin.
	\newblock Sparse reconstruction by convex relaxation: Fourier and Gaussian
	measurements.
	\newblock In {\em 2006 40th Annual Conference on Information Sciences and
		Systems}, pages 207--212. IEEE, 2006.
	
\end{thebibliography}
\end{document}