\documentclass[11pt]{article}
\usepackage{hyperref}
\usepackage{amsmath,amssymb,amsthm}
\usepackage{mathtools}  
\usepackage{graphicx}

\DeclareMathOperator*{\E}{\mathbb{E}}
\let\Pr\relax
\DeclareMathOperator*{\Pr}{\mathbb{P}}
\DeclareMathOperator*{\C}{\mathbb{C}}

\newcommand{\eps}{\epsilon}
\newcommand{\inprod}[1]{\left\langle #1 \right\rangle}
\newcommand{\R}{\mathbb{R}}
\DeclarePairedDelimiter{\abs}{\lvert}{\rvert}
\DeclarePairedDelimiter{\norm}{\lVert}{\rVert}
\DeclarePairedDelimiter{\bignorm}{\Arrowvert}{\Arrowvert}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {\bf CS 395T: Sublinear Algorithms } \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[4]{\handout{#1}{#2}{#3}{Scribe: #4}{Lecture #1}}

\newtheorem{theorem}{Theorem}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{observation}[theorem]{Observation}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{assumption}[theorem]{Assumption}

% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex

\begin{document}

\lecture{20 --- November 3, 2016}{Fall 2016}{Prof.\ Eric Price}{Yitao Chen, Shanshan Wu}
\section{Overview}
In this lecture, we are going to talk about Fourier RIP matrices, which includes the following
\begin{itemize}
\item Fourier Uncertainty Principle
\item Proving RIP
	\begin{itemize}
		\item Symmetrization/Gaussianization
		\item Chaining/Dudley's Entropy Integral
		\item Maurey's Emprical Method
	\end{itemize}
\end{itemize}
\section{Fourier Uncertainty Principle}
For $x \in \C^n$, the Discrete Fourier Transform (DFT) $\hat{x} \in \C^n$ is $\hat{x}=Fx$, where $F_{ij}=\omega^{ij}, \forall i,j$ and $\omega=e^{2\pi\sqrt{-1}/n}$, which implies $\omega^n=1$,
\begin{flalign*}
F=\begin{bmatrix}
    1 & 1 & 1 & \dots & 1 \\
    1 & \omega & \omega^2 & \dots & \omega^{n-1}  \\
    1 & \omega^2 & \omega^4 & \dots & \omega^{2(n-1)}\\
    \vdots & \vdots & \vdots & \ddots & \vdots \\
    1 & \omega^{n-1} & \omega^{2(n-1)} & \dots & \omega^{(n-1)^2}
\end{bmatrix}.
\end{flalign*}
For $F$ to be RIP matrix, we just need: $F\cdot F^*=nI_n$ and $\abs{F_{ij}}\le 1, \forall i,j$. Also notice that Hadamard matrix $H_{i,j}=(-1)^{<i,j>}$, where $i,j$ are vectors in $\{0,1\}^{\log n}$ has this property: $HH^T=nI_n$.

\begin{lemma}
Let $x \in \C^n$ be $k$-sparse, then we have $\text{supp}(\hat{x}) \ge n/k$.
\end{lemma}
\begin{proof}
Let $x^{(l)}$ be a ``modulation'' of $x$ ($x\neq 0$), $x_i^{(l)}=x_i\cdot \omega^{-li}$. Take a look at the $j$th coordinate of the Fourier transform of the modulated signal $x^{(l)}$, we have
\begin{flalign}\label{equ:rotate}
(\widehat{x^{(l)}})_j=(Fx^{(l)})_j=\sum_{i=0}^{n-1}\omega^{ij}x_i^{(l)}=\sum_{i=0}^{n-1}x_i\cdot \omega^{i(j-l)}=\hat{x}_{j-l}.
\end{flalign}
Let $X=\text{span}(x^{(0)}, x^{(1)}, \ldots, x^{(n-1)})$, Equation~(\ref{equ:rotate}) tells us the Fourier transform of $X$ just rotates the coordinates of $X$, and modulation does not change the sparseness, so
\begin{flalign}\label{equ:2}
k \ge \text{dim} (X)=\text{dim}(FX).
\end{flalign}
Let $T_l:=\cup_{j=0}^{l}\text{supp}(\widehat{x^{(j)}})$ and $S:=T_D=\text{supp}(\hat{x})$, we have $\abs{T_n}=n$ and 
\begin{flalign*}
\abs{T_l\setminus T_{l-1}} \le \abs{\text{supp}(\widehat{x^{(l)}})}=\abs{\text{supp}(\hat{x})}=\abs{S}.
\end{flalign*}
We claim 
\begin{flalign}\label{equ:3}
\text{dim}(FX) \ge \text{number of times $T_l \neq T_{l-1}$} \ge \frac{n}{|S|}.
\end{flalign}
Combining Equation~(\ref{equ:2}) and~(\ref{equ:3}), we have $\abs{S} \ge n/k$.
\end{proof}

\section{Proving RIP}
Let $F$ be unitary and bounded. Let $\Omega \subseteq [n]$ be a multiset of $m=O(\frac{k}{\epsilon^2}\log^4 n)$ i.i.d. uniform indices. We have the following claim,
\begin{claim}
$F_{\Omega}=$ rows of $F$ corresponding to $\Omega$ has $(k,\epsilon)$ RIP ``in expectation''. Here ``in expectation'' means considering
\begin{flalign*}
\Delta:=\E_{\Omega}\Bigg[\underset{S \subseteq [n], \abs{S}\le k}{\text{sup}} \norm{I-\frac{1}{m}F_{\Omega\times S}^TF_{\Omega\times S}}\Bigg].
\end{flalign*}
\end{claim}
Let $x_i^T$ be rows of $F$, so $\Delta$ can be written as
\begin{equation}
\Delta:=\E_{\Omega}\Bigg[\underset{S \subseteq [n], \abs{S}\le k}{\text{sup}} \norm{I-\frac{1}{m}\sum_{i\in \Omega}x_i^{S}{x_i^{S}}^{T}}\Bigg]. \label{delta}
\end{equation}
\subsection{Symmetrization/Gaussianization}
Let $\norm{\cdot}$ be a norm (it has convexity and triangle inequality holds) and $x_i \sim X$ independently for any $i$, we claim the following inequality holds,
\begin{flalign*}
\underset{x_1,\ldots,x_m \sim X}{\E}\Bigg[\norm{\frac{1}{m}\sum_{i=1}^{m}x_i-\E_{x \sim X}[x]}\Bigg] &\le 2\underset{x_1,\ldots,x_m \sim X, s_1,\ldots,s_m \sim \{\pm 1\}}{\E}\Bigg[\norm{\frac{1}{m}\sum_{i=1}^{m}s_ix_i}\Bigg]\\
&\le \sqrt{2\pi}\underset{x_1,\ldots,x_m \sim X, g_1,\ldots,g_m \sim N(0,1)}{\E}\Bigg[\norm{\frac{1}{m}\sum_{i=1}^{m}g_ix_i}\Bigg]
\end{flalign*}
\begin{proof}
Draw $x_i' \sim X$ independently, let $s_1,\ldots, s_m\sim \{\pm 1\}$, and $g_1,\ldots,g_m \sim N(0,1)$, we have
\begin{flalign*}
\text{LHS} &= \E_{x}\Bigg[\norm{\frac{1}{m}\sum_{i=1}^{m}x_i-\E_{x'}\bigg[\frac{1}{m}\sum_{i=1}^{m}x_i'}\bigg]\Bigg]\\
&\le \E_{x,x'} \Bigg[\norm{\frac{1}{m}\sum_{i=1}^{m}(x_i-x_i')}\Bigg] \\
&= \E_{x,x',s} \Bigg[\norm{\frac{1}{m}\sum_{i=1}^{m}s_i(x_i-x_i')}\Bigg] \\
&\le 2\E_{x,s}\Bigg[\bignorm{\frac{1}{m}\sum_{i=1}^{m} s_i x_i}\Bigg] \\
& = 2\E_{x,s}\Bigg[\bignorm{\frac{1}{m}\sum_{i=1}^{m} s_i x_i \E[\abs{g_i}]\cdot\sqrt{\frac{\pi}{2}}}\Bigg]\\
& \le \sqrt{2\pi} \E_{x,s,g}\Bigg[\bignorm{\frac{1}{m}\sum_{i=1}^{m} s_i\abs{g_i}x_i}\Bigg]\\
& = \sqrt{2\pi} \E_{x,g}\Bigg[\bignorm{\frac{1}{m}\sum_{i=1}^{m} g_ix_i}\Bigg].
\end{flalign*}
This first and the third inequalities follow from the convexity of norm operation. The second inequality follows from triangle inequality. The last equality is true because $s_i |g_i|\sim N(0,1)$.
\end{proof}
Next, let $\Sigma_k$:=$\{y \in \C^n: y \text{ is $k$-sparse and} \norm{y}_2=1\}$, we can use the Symmetrization/Gaussianization technique to transform (\ref{delta}) as 
\begin{align}
m\Delta &\lesssim \E_{\Omega,g} \text{sup}_{|S|\le k} \norm{\sum_{i\in\Omega} g_ix_i^S{x_i^S}^T}\nonumber\\
&\lesssim \E_{\Omega,g} \text{sup}_{y \in \Sigma_k} \abs{\sum_{i \in \Omega} g_i \inprod{x_i,y}^2}. \label{delta2}
\end{align}

We will now bound the RHS of the (\ref{delta2}) for every set $\Omega$ via a technique called {\em chaining}. Before introduction to chaining, let us first define a related concept called ``Gaussian Process"\footnote{We found this blog post~\cite{JN16} by Jelani Nelson very useful for understanding chaining methods.}. 

\section{Gaussian Process}
Suppose we are given a bounded (but may be infinite) set of vectors $S\subset \R^n$ (e.g., $S$ can be all $k$-sparse vectors with $\norm{\cdot}_2\le 1$). For each vector $x\in S$, we associate it with a random (usually zero-mean) Gaussian variable $G_x$. This is called a Gaussian Process. One property about $\{G_x\}_{x\in S}$ is that we can derive concentration bounds on $G_x-G_y$ in terms of a distance measure $d(x,y)$. More specifically, define $d(x,y) = \E[(G_x-G_y)^2]^{1/2}$, then $G_x-G_y$ is distributed as $N(0, d(x,y)^2)$.

As an example, let $g\in \R^n$ be a random vector with entries drawn i.i.d. from standard normal distribution. We can define $G_x:=\inprod{g,x}$. Then
$$
d(x,y)^2= \E[(G_x-G_y)^2] =  \E[(\inprod{g, x-y})^2] = \norm{x-y}_2^2, \quad \forall x, y\in S.
$$
Since $G_x-G_y$ is distributed as $N(0, d(x,y)^2)$, we can bound it as
$$
\Pr(\abs{G_x-G_y}\ge t) \lesssim e^{-\frac{t^2}{2\norm{x-y}_2^2}}.
$$

One nice property about Gaussian Process is that we can use chaining technique (which we will show in the next section) to bound 
\begin{equation}
\E[\sup_{x\in S} G_x].\label{sup}
\end{equation}
Many problems can be transformed into this form of (\ref{sup}). For example, consider the problem of computing $\E[\norm{A}_2]$ for a random matrix $A\in \R^{m\times n}$ with entries being i.i.d. $N(0,1)$. This problem can be formulated as 
$$
\E[\sup_{x\in \mathbb{S}^{n-1},\; y\in \mathbb{S}^{m-1}} y^T Ax],
$$
where $\mathbb{S}^{n-1} = \{x\in \R^n: \norm{x}_2 = 1\}$ is the $n$-dimensional unit sphere. In this case, we define a Gaussian Process on the set $S = \{(x,y): x\in \mathbb{S}^{n-1},\; y\in \mathbb{S}^{m-1}\},$ with distance metric given by
\begin{align}
d((x,y), (x',y'))^2 &= \E[(y^TAx-y'^TAx')^2] =  \E[(\text{tr}(y^TAx)-\text{tr}(y'^TAx'))^2] \nonumber\\
& = \E[(\text{tr}(Axy^T)-\text{tr}(Ax'y'^T))^2] = \E[(\text{tr}(Axy^T)-\text{tr}(Ax'y'^T))^2]  \nonumber\\
& = \E[(\text{tr}(A(xy^T-x'y'^T))^2] =  \norm{xy^T-x'y'^T}_F^2,\nonumber
\end{align}
where the last equality follows from the fact each entry of $A$ is i.i.d. standard Gaussian. 
\section{Chaining}
Given a set $S$ and a Gaussian process $\{G_x\}_{x\in S}$, we are interested in bounding $\E[\sup_{x\in S} G_x]$. We show how chaining techniques can be used to obtain a non-trivial bound.
\subsection{Naive bound}
In Homework 1, we have shown that the maximum value of $n$ i.i.d. N(0,1) random variables scales as $\sqrt{\log n}$. The same technique (for upper bound) can be used here to derive a bound when $S$ is a finite set. Suppose $0\in S$, $G_0 = 0$, then we have\footnote{A detailed proof can be found in~\cite{JN16}.}
\begin{equation}
\E[\sup_{x\in S} G_x] \lesssim \max_{x\in S} d(x,0) \sqrt{\log |S|}, \label{naive}
\end{equation}
where $d(x,0)=\E[G_x^2]^{1/2}$ captures the radius of $S$ under the given Gaussian process. The above bound depends on $|S|$, and hence is not applicable if $S$ is a infinite set.

\subsection{An $\eps$-cover}
Consider an $\eps$-cover of $S$ under the distance metric $d$,  i.e., for every $x\in S$, there exists a vector $c(x)$ such that $d(x, c(x))\le \eps$. Let $\mathcal{N}(S, d, \eps)$ be the covering number. Since $G_x = G_x - G_{c(x)} + G_{c(x)}$, we have 
\begin{align}
\E[\sup_{x\in S} G_x] &\le \E[\sup_{x\in S} G_x - G_{c(x)}] + \E[\sup_{x\in S} G_{c(x)}] \nonumber\\
& \lesssim \eps \sqrt{\log |S|} + \max_{x\in S} d(c(x),0) \sqrt{\log \mathcal{N}(S, d, \eps)}, 
\end{align}
where the last inequality follows from (\ref{naive}) and $d(x-c(x), 0) = d(x, c(x)) \le \eps$. The above bound is better than (\ref{naive}), however, it still depends on $|S|$. To remove this dependence, we can use the same idea repeatedly by constructing a sequence of $\eps$-covers, a technique called chaining.

\subsection{Chaining}
The idea of chaining is to partition $S$ into a sequence of covers at different resolutions, and then use (\ref{naive}) to bound their difference. Let $R = \max_{x\in S} d(x,0)$ be the radius $S$ under distance metric $d$. Let $S_i \subset S$ be a $R/2^i$-cover of $S$. For every $x\in S$, let $x_i\in S_i$ be the closest point to $x$, then $d(x, x_i)\le R/2^i$. Suppose $S_0 = \{0\}$, then $x_0 = 0$. We have
$$
G_x = G_{x_0} + G_{x_1}-G_{x_0}+G_{x_2}-G_{x_1}+\cdot\cdot\cdot = \sum_{r=1}^{\infty} G_{x_r} -G_{x_{r-1}} = \sum_{r=1}^{\infty} G_{x_r-x_{r-1}}.
$$
Let $\mathcal{N}(S, d, R/2^i)$ be the corresponding covering number, and suppose that $|S_i| = \mathcal{N}(S, d, R/2^i)$. Then
\begin{equation}
\E[\sup_{x\in S} G_x] \le \sum_{r=1}^{\infty} \E[\sup_{x\in S}G_{x_r-x_{r-1}}]
\lesssim \sum_{r=1}^{\infty} \frac{R}{2^r} \sqrt{\log\mathcal{N}(S, d, \frac{R}{2^r})^2} \lesssim \int_{0}^{\infty} \sqrt{\log\mathcal{N}(S, d, u)} \text{d}u,\label{dud}
\end{equation}
where the second inequality uses (\ref{naive}), with triangle inequality 
$$
d(x_r, x_{r-1}) \le d(x, x_r)+ d(x, x_{r-1}) \le \frac{R}{2^r} + \frac{R}{2^{r-1}} = \frac{3R}{2^r},
$$
being used to bound  $\max_{x\in S} d(x_r, x_{r-1})$. Note that $|\{x_r-x_{r-1}: x\in S\}| \le \mathcal{N}(S, d, \frac{R}{2^r})^2$. 

The bound given in (\ref{dud}) is called {\em Dudley's Entropy Integral}. How good is this bound? It can lose a $\log $ factor. An even tighter bound can be obtained using the generic chaining method~\cite{Fer75, Tal14}.  

\bibliographystyle{alpha}

\begin{thebibliography}{42}

\bibitem[JN16]{JN16}
Jelani Nelson.
\newblock Chaining methods continued.
\newblock Blog post. URL: \url{https://windowsontheory.org/2016/02/29/chaining-methods-continued-guest-post-by-jelani-nelson/}, February, 2016.

\bibitem[Dud67]{Dud67} 
Richard M. Dudley. 
\newblock The sizes of compact subsets of Hilbert space and continuity of Gaussian processes. 
\newblock {\em J. Functional Analysis}, 1:29--330, 1967.

\bibitem[Fer75]{Fer75} 
Xavier Fernique. 
\newblock Regularité des trajectoires des fonctions aléatoires gaussiennes. 
\newblock {\em Lecture Notes in Math.}, 480:1--96, 1975.

\bibitem[Tal14]{Tal14} 
Michel Talagrand. 
\newblock Upper and lower bounds for stochastic processes: modern methods and classical problems. 
\newblock {\em Springer}, 2014.

\end{thebibliography}

\end{document}