\documentclass[11pt]{article}
\usepackage{amsmath,amssymb,amsthm}
\usepackage{framed}


\DeclareMathOperator*{\E}{\mathbb{E}}
\let\Pr\relax
\DeclareMathOperator*{\Pr}{\mathbb{P}}

\newcommand{\eps}{\epsilon}
\newcommand{\inprod}[1]{\left\langle #1 \right\rangle}
\newcommand{\R}{\mathbb{R}}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {\bf CS 395T: Sublinear Algorithms } \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[4]{\handout{#1}{#2}{#3}{Scribe: #4}{Lecture #1}}

\newtheorem{theorem}{Theorem}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{observation}[theorem]{Observation}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{assumption}[theorem]{Assumption}
\newenvironment{algorithm}[1][\unskip]
{\begin{framed}\textbf{Algorithm: #1}}
{\end{framed}}

% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex

\begin{document}

\lecture{19 --- Nov 1, 2016}{Fall 2016}{Prof.\ Eric Price}{Changyong Hu and Zhao Liu}

\section{Overview}

Last lecture, we talked about adaptive sparse recovery. Today, we will talk about RIP-1, expanders and SSMP.


\section{Sparse Matrix with RIP-1}
Recalled from homework 3, we showed that 0-1 matrices that satisfy the RIP-2 cannot be very sparse. Alternatively, there exists a lower bound $m = \Omega(k^2)$ that an 0-1 matrix $A\in \R^{m \times n}$ with $d =O(logn)$ ones per column satisfies RIP-2. But A can satisfy the following RIP-1 property \cite{BGIKS08}. Today we will show deterministic sparse recovery and fast-embedding via sparse matrices that satisfy RIP-1 property.

\begin{definition}
$A$ has RIP-1 of $(k,\epsilon)$ if $\forall $k-sparse $x$, $ (1-\epsilon) \|x\|_1 \le \| Ax\|_1 \le \|x\|_1 $.
\end{definition}

\begin{definition}
$G = (U,V,E)$ is a bipartite graph with left-degree $d$. $n = |U|$, $m = |V|$. $N(S)$ denotes the neighbors of $S$. $G$ is a unbalanced bipartite expander of $(k,\epsilon)$ if
\begin{quote}
$\forall S \subset U , |S| \le k \Rightarrow N(S) \ge (1-\epsilon)d|S|$
\end{quote}
\end{definition}

\begin{claim}
A random sparse binary matrix with $m = \frac{1}{\epsilon^2}klog\frac{n}{k}, d = \frac{1}{\epsilon}log\frac{n}{k}$ satisfies RIP-1(after scaling by $\frac{1}{d}$, d is degree of sparse matrix that is number of ones by column).
\end{claim}

\begin{claim}
More generally, an adjacency matrix of expander graph $G(n,m,k,d,\epsilon)$ has $(k,2\epsilon)$ RIP-1.
 
\end{claim}




\begin{theorem}
A random sparse binary matrix with $m = \frac{1}{\epsilon^2}klog\frac{n}{k}, d = \frac{1}{\epsilon}log\frac{n}{k}$ is a  $(k, 2\epsilon)$ expander with high probability.
\end{theorem}

\begin{proof}
$\forall S$ of size $k$, $m = \frac{1}{\epsilon^2} k log \frac{n}{k}$, $d = \frac{1}{\epsilon} log \frac{n}{k}$. $N(S)$ has $dk$ balls in $m$ bins. Consider all $kd$ edges, define $V_1, V_2, \cdots, V_{kd} \in [m]$ and be in $i.i.d$. Let $E_i$ denotes the event that any ball $V_i$ collides with previous balls, that collides with $V_1, V_2, \cdots, V_{i-1}$. We have
  $$
    Pr[E_i] \le \frac{i-1}{m} \le \frac{kd}{m} \le \epsilon \\
  $$
  $$
    Pr[\sum_{j=1}^{kd} E_j > \epsilon kd] \le e^{-\Omega(\epsilon dk)}\\  
  $$
  $$
  	Pr[|N(S)| \le (1-2\epsilon)dk] \le e^{-\Omega(\epsilon dk)}\\ 
  $$
Set $d = O(\frac{1}{\epsilon} log \frac{n}{k}$) \\
\begin{align*}
\Rightarrow Pr[|N(S)| \le (1-2\epsilon)dk] &\le e^{-\Omega(k log \frac{n}{k})} \\
& \le \frac{1}{(\frac{k}{n})^{2k}}\\
& \le \frac{1}{\binom nk^2}
\end{align*}
By union bound, all sets $S$ of size $k$ expands with $1-\frac{1}{\binom nk}$ probability. For sets of size less than $k$, if size is $k'$, it still works with $1-\frac{1}{\binom n{k'}}$. So by union bound over $k'$, all sets of size $\le k$ expands with good probability.
\end{proof}

\begin{claim}
There exists explicit expander constructions for $\forall \alpha > 0, d = logn \cdot  ({\frac{logk}{\epsilon}})^{1+\frac{1}{\alpha}}, m = k^{1+\alpha}d^2 $.
\end{claim}

\section{L1 minimization with RIP-1}
If we have RIP-1 matrices, we can use L1 minimization to do sparse recovery.\\
Given $y=Ax+e$, pick $\hat{x}=argmin \|\hat{x}\|_1  \ s.t. \|A\hat{x}-y\|_1 \le \Delta $.\\
\begin{theorem}
if A has (k,$2\epsilon$) RIP-1, $\|e\|_1 \leq \Delta$, then we have $\|\hat{x}-x\|_1\lesssim 2\Delta$.
\end{theorem}

Set $z=\hat{x}-x$, we have $\|Az-e\|_1 = \|A(\hat{x}-x)-e\|_1 = \|A\hat{x}-y\|_1 \le \Delta$. By triangle inequality, we have $\|Az\|_1 \le \|Az\|_1 + \|e\|_1 \le 2\Delta$. We only need $\|z\|_1 \lesssim \|Az\|_1$, so that $\|\hat{x}-x\|_1\lesssim 2\Delta$ follows.


\begin{lemma}
if $Az=0$, $\forall |S|=k$, then $\|z_s\|_1 \le \frac{2\epsilon}{1-2\epsilon}\|z\|_1$.
\end{lemma}

\begin{proof}
Partition [$n$]  into $S_0 \bigcup S_1 \bigcup \cdots \bigcup S_L, |S_l|=k$ in decreasing order of $z_i$, and $S=S_0$. $m = \frac{1}{\epsilon^2} k log \frac{n}{k}$, $d = \frac{1}{\epsilon} log \frac{n}{k}$. Then $|N(S)|\approx d|S|=dk=O(\epsilon m)$.\\
Pick $A'= \text{rows of A cooresponding to } N(S)$, $Az = 0 \Rightarrow A'z=0$.
\begin{align*}
0 &= \|A'z\|_1 \\
&= \|A'z_s + \sum_{l \ge 1}(A'z_{s_l})\|_1\\
& \ge \|A'z_s\|_1 - \|\sum_{l \ge 1}(A'z_{s_l})\|_1 \text{ (by triangle inequality)}\\
& \ge \|A'z_s\|_1 - \sum_{l \ge 1}\|A'z_{s_l}\|_1 \text{ (by triangle inequality)}\\
& \ge (1-2\epsilon)\cdot d \cdot \|z_s\|_1 - \sum_{l \ge 1}(\# \text{ edges from } S_l \text{ to } N(S)) \cdot \max_{i \in s_l}|z_i| \text{ (by definition of RIP-1)}\\
& \E[\# \text{ edges from } S_l \text{ to } N(S)] = \frac{d}{m} \cdot |N(S)| \cdot k < \frac{dk}{m}\cdot dk  < \frac{dk}{m} \cdot m\epsilon = \epsilon mk \text{(by $dk < \epsilon m$)}\\
& w.h.p [\# \text{ edges from } s_l \text{ to } N(S)] < 2\epsilon mk \\
& \ge (1-2\epsilon)\cdot d \cdot \|z_s\|_1 - 2\epsilon mk \cdot \frac{\sum_{l \ge 1}\|z_{s_{l-1}}\|_1}{k} \text{(by decreasing order of $z_i$)} \\
& \ge (1-2\epsilon)\cdot d \cdot \|z_s\|_1 - 2 \epsilon \cdot d \cdot \|z\|_1 
\end{align*}
$\Rightarrow \|z_s\|_1 \le \frac{2\epsilon}{1-2\epsilon}\|z\|_1$
\end{proof}


\section{Sequential Sparse Matching Pursuit}
Suppose $x$ is sparse, given $y=Ax$ ($A$ is random sparse RIP-1 binary matrix), for each $i$, how to estimate $x_i$? We can minimize $\|y-A(\hat{x_i}\cdot e_i)\|_1$, it turns out that $\hat{x_i} = \underset{N(i)}{\mathrm{median}}(y_i)$ (similar to 'count-sketch').\\

\begin{algorithm}[SSMP] \\
 1) Let $x^{(0)} = 0$ \\
 2) For $r = 1,2, \cdots, T = O(log(\|x \|_1/\|e\|_1))$  
    \begin{quote}
	a) For $ t = 1,2,\cdots, 10k$  
	\begin{itemize}
	    \item $\hat{x_i} \leftarrow \underset{j \in N(i)}{median} (y-Ax^{(r)})_j$
	    \item Let $i$ be the largest term of $\hat{x}$ 
	    \item Let $x^{(r)} \leftarrow x^{(r)} + \hat{x_i} e_i$
	\end{itemize}
	b) Let $x^{(r+1)} = H_k(x^{(r)})$ (the top $k$ values of $x^{(r)}$)
    \end{quote}
 3) Report $x' = x^{(T)}$
\end{algorithm}

The idea of proving SSMP is using the Lemma below: 
  Given $y = Ax$,\\
  
  \begin{lemma}
    There exists $i$ s.t. $\|y-A\hat{x_i}\cdot e_i \|_1 \le (1-\frac{1}{10k}) \| y\|_1$
  \end{lemma}
  Therefore each step $x^{(r)} = x^{(r)} + \hat{x_i} e_i$ decreases $\| y - Ax^{(r)}\|$ by $1-\frac{\Omega(1)}{k}$, after $O(k)$ steps, we have
  $$\|y-Ax^{(r)}\| \le \frac{1}{10} \| y - Ax^{(r-1)}\| $$
  repeat $log(\frac{\|x\|_1}{\epsilon})$ times converges to $\|y-Ax^*\|_1 \le \epsilon$.\\
  A more detailed proof can be found at \cite{BI09}.
  
  
  How fast is SSMP? It will take $log(\frac{\|x\|_1}{\epsilon})$ times sequentially add $O(k)$ single terms to minimize $\|y-Ax_ie_i\|_1$.\\
  Naive update will take $O(knd)$ per loop. Can do better with $O(dnlog(n))$\\
  We can spend $nd$ times for the first update and will find all the neighbors in $y$. For every successive time, it modifies $d$ elements of $y$ by looking at all neighbors of these. \\
  $\Rightarrow \frac{d^2 n}{m} \approx \frac{dn}{k} log(n) $.



\bibliographystyle{alpha}

\begin{thebibliography}{42}


\bibitem[BGIKS08]{BGIKS08}
Radu Berinde and Anna C. Gilbert and Piotr Indyk and Howard J. Karloff and Martin J. Strauss.
\newblock Combining geometry and combinatorics: {A} unified approach to sparse signal recovery.
\newblock {\em CoRR}.

\bibitem[BI09]{BI09}
Berinde, Radu, and Piotr Indyk. 
\newblock Sequential sparse matching pursuit.
\newblock {\em Communication, Control, and Computing, 2009. Allerton 2009. 47th Annual Allerton Conference on. IEEE}, 2009.



\end{thebibliography}

\end{document}
