\documentclass[11pt]{article}
\usepackage{amsmath,amssymb,amsthm}
\usepackage{framed}


\DeclareMathOperator*{\E}{\mathbb{E}}
\let\Pr\relax
\DeclareMathOperator*{\Pr}{\mathbb{P}}

\newcommand{\eps}{\epsilon}
\newcommand{\inprod}[1]{\left\langle #1 \right\rangle}
\newcommand{\R}{\mathbb{R}}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {\bf CS 395T: Sublinear Algorithms } \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[4]{\handout{#1}{#2}{#3}{Scribe: #4}{Lecture #1}}

\newtheorem{theorem}{Theorem}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{observation}[theorem]{Observation}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{assumption}[theorem]{Assumption}
\newenvironment{algorithm}[1][\unskip]
{\begin{framed}\textbf{Algorithm: #1}}
{\end{framed}}

% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex

\begin{document}

\lecture{14 --- Oct 14, 2014}{Fall 2014}{Prof.\ Eric Price}{Xiangru Huang}

\section{Overview}

In the last lecture : lower bounds

In this lecture: problems for final project, RIP-1, expanders, SMP, SSMP

\section{Problems for final project}
\begin{itemize}
\item {\bf Adaptivity in Sparse Recovery} \\ 
	So far, we can choose matrix $A$ independent of $x$ and estimate $x$ from $Ax$ using $O(klog\frac{n}{k})$ space.
	What if we choose $<v_1, x>,~<v_2,x>,~\cdots,~<v_m,x>$ where $v_i$ depend on $v_1,~v_2,~\cdots,~v_{i-1}$?

\item {\bf k-sparse} \\
	Given $A \sim \mathcal{N}(0,I_{M \times N})$, we know that if $m$ = $O(k log \frac{n}{k})$ (hence RIP), then L1 minimization and IHT works. \\
	But for $m << k log\frac{n}{k}$, doing $\ell_1/\ell_1$ recovery is impossible. 
	What if $x$ is exactly $k$-sparse?

\item {\bf Compressed Sensing with priors} \\
	e.g. If x follows some distribution, what could happen? This is a general question.
	
\item {\bf Count-sketch} \\
	Using top $2k$ coordinates, we can do $(1+\epsilon)~ \ell_2/\ell_2$ approximate recovery where $m = O(\frac{k}{\epsilon} logn)$
	If we use top $k$ coordinates, then $m = O(\frac{k}{\epsilon^2}logn)$. \\
	Better analysis used in \cite{MP14} might give $O(\frac{k}{\epsilon^2}+\frac{k}{\epsilon}logn)$.
	
\item {\bf LASSO vs sqrt LASSO} \\
	LASSO finds $\underset{x}{argmin} \| y - Ax \|^2_2 + \lambda \|x\|_1$. And sqrt LASSO finds $\underset{x}{argmin} \| y - Ax \|_2 + \lambda \|x\|_1$.
	Compare these two algorithms. Which one is better under certain situations?
	
\item {\bf Random order streams} \\
	Given $x_1,~x_2,~\cdots,~x_m \sim D$ over $F_2^n$.
	e.g. $x = Ay$ (mod 2) where $A$ is a $n$ by $\frac{n}{2}$ matrix.
	One of the two cases is true:
	\begin{itemize}
	 \item $D$ is $\frac{n}{2}$-dimension subspace
	 \item $D$ is uniform
	\end{itemize}
	How many samples to distinguish these two cases?
\end{itemize}

\section{Sparse Matrix with RIP-1}
Consider 0-1 matrix $A\in \R^{m \times n}$ with $d =O(logn)$ ones per column. For $A$, we can achieve fast multiplication and $O(logn)$ update time.
The problem is $A$ can't satisfy RIP unless $m = O(k^2)$. But $A$ can satisfy the following RIP-1 property \cite{BGIKS08}.

\begin{definition}
$A$ has RIP-1 of $(k,\epsilon)$ if $\forall $k-sparse $x$, $\| Ax\|_1 = (1\pm\epsilon) \| x\|_1$.
\end{definition}

\begin{definition}
$G = (U,V,E)$ is a bipartite graph with left-degree $d$. $n = |U|$, $m = |V|$. $N(S)$ denotes the neighbors of $S$. $G$ is a unbalanced bipartite expander of $(k,\epsilon)$ if
\begin{quote}
$\forall S \subset U , |S| \le k \Rightarrow N(S) \ge (1-\epsilon)d|S|$
\end{quote}
\end{definition}

\begin{claim}
 random graph with $d = \frac{1}{\epsilon}log\frac{n}{k}, m = \frac{1}{\epsilon^2}klog\frac{n}{k}$ is expander w.h.p.
\end{claim}

\begin{claim}
 There exist explicit expander constructions for $\forall \alpha > 0, d = O(logn {\frac{logk}{\epsilon}})^{1+\frac{1}{\alpha}}, m = k^{1+\alpha}d^2 $
\end{claim}

The adjacency matrices of expander graphs, scaled by a factor of $\frac{1}{d}$, satisfy RIP-1.

\begin{theorem}
 $(k,\epsilon)$ expander $\Rightarrow (k,2\epsilon)$ RIP-1
\end{theorem}

\begin{proof}
 $\forall S$ of size $k$, $d = \frac{1}{\epsilon} log \frac{n}{k}$, $m = \frac{2}{\epsilon}kd$. 
 Consider all $kd$ edges, define $V_1, V_2, \cdots, V_{kd} \in [m] i.i.d$.
 Let $C_j$ denotes the event that $V_j$ collide with $V_1, V_2, \cdots, V_{j-1}$. We have
  $$
    Pr[C_j] \le \frac{j-1}{m} \le \frac{kd}{m} \le \frac{\epsilon}{2} \\
  $$
  $$
    Pr[\sum_{i=1}^{kd} C_i > \epsilon kd] \le 2^{-\frac{\epsilon kd}{2}} = 2^{-\Omega(\frac{\epsilon kd}{2})}
  $$
 By union bound, the probability that $N(S) \ge (1-\epsilon)d|S|$ holds for all $S \subset U, |S| = k$  is at least $1 - 2^{-\Omega(k log\frac{n}{k})} {n \choose k} = 1 - 2^{-\Omega(k log\frac{n}{k})}$.
 Then sum over all sizes we can show $\sum_{i=0}^{n} 2^{-ilog\frac{n}{k}} << 1$, which means the theorem is true w.h.p.
\end{proof}

\section{From RIP-1 to sparse recovery}

There is a viable natual algorithm from count-sketch. You can check out the explanation in the book \cite{FSR}.
\begin{algorithm}[Natural Alg] \\ 
  $x^{(0)} = 0$ \\
  For $r = 0,1,\cdots, T \{$ \\
  $~~~~u_i \leftarrow \underset{j \in N(i)}{median} (y-Ax^{(r)})_j$ \\
  $~~~~x^{(r+1)} \leftarrow H_k (x{(r)}+u)$ \\
  $\}$ \\
  output $x^{(T)}$
\end{algorithm}

Here we introduce SMP \cite{BIR08} and SSMP \cite{BI09}. The following algorithm is SMP.
\begin{algorithm}[SMP] \\ 
  $x^{(0)} = 0$ \\
  Repeat $T$ times  
  \begin{quote}
  $~~~~u_i \leftarrow \underset{j \in N(i)}{median} (y-Ax^{(r)})_j$ \\
  $~~~~x^{(r+1)} \leftarrow H_k (x{(r)}+H_{2k}(u))$ 
  \end{quote}
  output $x^{(T)}$
\end{algorithm}

SSMP is similar to SMP except the updates is done sequentially.
\begin{algorithm}[SSMP] \\
 1) Let $x^{(0)} = 0$ \\
 2) For $r = 1,2, \cdots, T = O(log(\|x \|_1/\|e\|_1))$  
    \begin{quote}
	a) For $ t = 1,2,\cdots, 10k$  
	\begin{itemize}
	    \item $u_i \leftarrow \underset{j \in N(i)}{median} (y-Ax^{(r)})_j$
	    \item Let $i$ be the largest term of $u$ 
	    \item Let $x^{(r)} = x^{(r)} + u_i e_i$
	\end{itemize}
	b) Let $x^{(r)} = H_k(x^{(r)})$
    \end{quote}
 3) Report $x' = x^{(T)}$
\end{algorithm}

Here we prove SSMP. A more detailed proof can be found at \cite{BI09}
\begin{proof}
  $y = Ax = (\sum_{i \in S} a_ix_i) + e$ \\
  If $\sum \| a_ix_i\|_1 \le (1+\epsilon) \| \sum a_i x_i\|_1$, \footnote{you will prove this in your homework} we can show $\exists a_i, i, s.t.$ 
  $$
    \|y-a_ix_i \|_1 \le (1-\frac{1}{10k}) \| y\|_1
  $$
  Therefore each step $x^{(r)} = x^{(r)} + u_i e_i$ decreases $\| y - Ax^{(r)}\|$ by $1-\frac{\Omega(1)}{k}$, after $O(k)$ steps, we have
  \begin{align*}
		&\|y-Ax^{r}\| \le \frac{1}{10} \| y - Ax^{(r-1)}\| \\
    \Rightarrow & \|x^{(r)}-x\|_1 \le \frac{1}{5} \| x^{(r)}-x \|_1 + O(\|e\|_1) \\
    \Rightarrow & It~ takes~ O(log(\|x \|_1/\|e\|_1)~iterations~ to~ get~ error~ to~ O(\|e\|_1) \\
  \end{align*}

\end{proof}


\bibliographystyle{alpha}

\begin{thebibliography}{42}

\bibitem[AMS99]{AMS99}
Noga~Alon, Yossi~Matias, Mario~Szegedy.
\newblock The Space Complexity of Approximating the Frequency Moments.
\newblock {\em J. Comput. Syst. Sci.

\bibitem[MP14]{MP14}
Gregory~T.~Minton and Eric~Price.
\newblock Improved Concentration Bounds for Count-Sketch
\newblock {\em SODA(best student paper)} 2014.}, 58(1):137--147, 1999.

\bibitem[BGIKS08]{BGIKS08}
Radu Berinde and Anna C. Gilbert and Piotr Indyk and Howard J. Karloff and Martin J. Strauss.
\newblock Combining geometry and combinatorics: {A} unified approach to sparse signal recovery.
\newblock {\em CoRR}.

\bibitem[BI09]{BI09}
Berinde, Radu, and Piotr Indyk. 
\newblock Sequential sparse matching pursuit.
\newblock {\em Communication, Control, and Computing, 2009. Allerton 2009. 47th Annual Allerton Conference on. IEEE}, 2009.

\bibitem[BIR08]{BIR08}
Berinde, Radu, Piotr Indyk, and Milan Ruzic. 
\newblock Practical near-optimal sparse recovery in the l1 norm.
\newblock {\em Communication, Control, and Computing, 2008 46th Annual Allerton Conference on. IEEE}, 2008.

\bibitem[FSR]{FSR}
Foucart, Simon, and Holger Rauhut.
\newblock A mathematical introduction to compressive sensing.
\newblock {\em Berlin: Springer, 2013.}

\end{thebibliography}

\end{document}
