\documentclass[11pt]{article}
\usepackage{amsmath,amssymb,amsthm}
\usepackage{float}
\usepackage{graphicx}
\usepackage{multirow}
\DeclareMathOperator*{\E}{\mathbb{E}}
\let\Pr\relax
\DeclareMathOperator*{\Pr}{\mathbb{P}}

\newcommand{\eps}{\epsilon}
\newcommand{\inprod}[1]{\left\langle #1 \right\rangle}
\newcommand{\R}{\mathbb{R}}
\newcommand{\norm}[2]{\| #1 \|_{#2}}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {\bf CS 395T: Sublinear Algorithms } \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[4]{\handout{#1}{#2}{#3}{Scribe: #4}{Lecture: #1}}

\newtheorem{theorem}{Theorem}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{observation}[theorem]{Observation}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{definition}{Definition}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{assumption}[theorem]{Assumption}
\newtheorem{exercise}{Exercise}


\newcommand{\supp}{\text{supp}}
\newcommand{\F}{\mathcal{F}}

% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex

\begin{document}

\lecture{16 -- $L^1$ minimization, Oct 20, 2016}{Fall 2016}{Prof.\ Eric Price}{David Liau and Ger Yang}

\section{Overview}
In the last lecture: {\em Model-based Compressed Sensing }.

In this lecture: {\em $L^1$ minimization}.

\section{$L^1$ minimization}
Given
\begin{equation*}
A \in R^{m\times n} \quad with \quad \big(O(k),1\big) RIP
\end{equation*}
For a k-sparse vector $x$, define $y = Ax+e$. The problem is to find $\hat{x}$ with $\norm{\hat{x}-x}{2} \lesssim \norm{e}{2}$. Now if we pick $\hat{x}$ as the solution to the following optimization problem,
\begin{equation*}
\begin{aligned}
& \underset{\hat{x}}{\text{minimize}}
& & \norm{\hat{x}}{1} \\
& \text{subject to}
& & \norm{A\hat{x}-y}{2} \leq \epsilon. \;
\end{aligned}
\end{equation*}
We have the theorem which,
\begin{theorem}\label{cs-l1}
If $\eps \geq \norm{e}{2}$ then
$\norm{\hat{x} - x}{2} \lesssim \eps$.
\end{theorem}
\begin{proof}
Lets set $z=\hat{x}-x$, by definition we have,
\begin{align*}
\norm{Az-e}{2}^2 &= \norm{A\hat{x}-y}{2}^2 \leq \epsilon^2 \\
\norm{Az}{2}^2 &- 2(Az)^T e+\norm{e}{2}^2 \leq \epsilon^2 \\
\norm{Az}{2}^2 &= 2(Az)^T e-\norm{e}{2}^2 + \epsilon^2 \\
               &\leq 2\norm{e}{2} \norm{Az}{2}+\epsilon^2-\norm{e}{2}^2\\
\big( \norm{Az}{2}&-\norm{e}{2} \big)^2 \leq \epsilon^2\\
\norm{Az}{2} &\norm{e}{2}+\epsilon \leq 2\epsilon
\end{align*}
Now we just need $\norm{z}{2} \leq \norm{Az}{2}$ to complete the proof. Fortunately, the property is known to be true given by the Restricted Eigenvalue property discussed in the following section. Thus we have proved that by utilizing the $L^1$ minimization vector, it is possible to construct the sparse vector we desired.
\end{proof}

\subsection{Restricted Eigenvalue (RE)}
For $S = \supp(x)$,
\begin{align*}
\norm{x_S}{1} = \norm{x}{1} & \geq \norm{\hat{x}}{1} \\
& = \norm{x+z}{1} \\
& \geq \norm{(x+z)_S}{1} + \norm{z_{\bar{S}}}{1} \\
& \geq \norm{x_S}{1} + \norm{z_{\bar{S}}}{1} - \norm{z_{S}}{1}
\end{align*}
Thus we have that
\begin{equation*}
\norm{z_{S}}{1} \geq \norm{z_{\bar{S}}}{1}.
\end{equation*}
The last equation states the following important property we would like to introduce in this section.
\begin{definition}{Uniform Restricted Eigenvalue Condition (RE)}\\
We said that the matrix $A$ satisfies the uniform restricted eigenvalue condition if for all $S$ and $z$ which $\norm{z_S}{1} \geq \norm{z_{\bar{S}}}{1}$ holds, $\norm{Az}{2} \gtrsim \norm{z}{2}$.
\end{definition}

\subsection{RIP $\Rightarrow$ RE via Shelling Argument}
In this section, we are going to show that RIP implies RE via a ``shelling argument''.
More precisely, suppose $A$ satisfies the RIP of order $k$.  We would like to show that	for any $z$ and $S \subset [n]$ with $|S| \le k$ such that $\norm{z_S}{1} \geq	\norm{z_{\overline{S}}}{1}$, we have $\norm{Az}{2} \gtrsim \norm{z}{2}$.

The \emph{shelling argument} works as follows.  First, we split $z$ into blocks $z^1, z^2, \dots$ of size $k$ in the order of decreasing magnitude, i.e. $z=z^1 + z^2 + \dots + z^{n/k}$, and $\|z^1\|_2 \ge \|z^2\|_2 \ge \dots \ge \|z^{n/k}\|_2$.  Then, for $i \geq 2$ we have that
\begin{equation} \label{eq:rel1}
\frac{\norm{z^i}{1}}{\sqrt{k}} \leq \norm{z^i}{2} \leq
\frac{\norm{z^{i-1}}{1}}{\sqrt{k}}
\end{equation}
Also, for $i \ge 3$ we have
\begin{equation} \label{eq:rel2}
\norm{z^i + z^{i+1}}{2} \le
\frac{\norm{z^{i-1} + z^{i-2}}{1}}{\sqrt{2k}}
\end{equation}

Next, we look at $\norm{Az}{2}$:

\begin{align}
\norm{Az}{2} & = \norm{A(z^1 + z^2 + \dots + z^{n/k})}{2} \nonumber\\
& \geq \norm{A(z^1+z^2+z^3)}{2} - \norm{A(z^4+z^5)}{2} - \norm{A(z^6+z^7)}{2} - \dots \nonumber\\
& \geq (1-\eps)\norm{z^1 + z^2 + z^3}{2} - (1+\eps)\left(\sum_{i=2}^{\frac{n/k-1}{2}} \norm{z^{2i}+z^{2i+1}}{2}\right) \label{eq:az1}
\end{align}
where the first inequality is a result of triangle inequality, and the second one uses the RIP property.  Then, we can further bound the second term in \eqref{eq:az1} by
\begin{align*}
(1+\eps)\left(\sum_{i=2}^{\frac{n/k-1}{2}} \norm{z^{2i}+z^{2i+1}}{2}\right)
&\le \frac{(1+\eps)}{\sqrt{2k}}\left(\sum_{i=2}^{\frac{n/k-1}{2}} \norm{z^{2i-1}+z^{2i-2}}{1}\right) \\
&= \frac{(1+\eps)}{\sqrt{2k}}\sum_{i=2}^{\frac{n/k-1}{2}} \left( \norm{z^{2i-1}}{1}+\norm{z^{2i-2}}{1}\right) \\
&\le \frac{(1+\eps)}{\sqrt{2k}}\left\| \sum_{i=2}^{n/k} z^i \right\|_1 \le \frac{(1+\eps)}{\sqrt{2k}}\left\| z^1 \right\|_1
\end{align*}
where the first inequality uses \eqref{eq:rel2}.  The equality in the second line and the first inequality in the third line follows the fact that each $z^i$ has distinct support.  The last inequality is a result of the assumption that for any $|S| \le k$ such that $\norm{z_S}{1} \geq	 \norm{z_{\overline{S}}}{1}$, we have $\norm{Az}{2} \gtrsim \norm{z}{2}$.  Then, we are able to get a bound of \eqref{eq:az1}.
\begin{align*}
\norm{Az}{2} &\ge (1-\eps)\norm{z^1 + z^2 + z^3}{2} - (1+\eps)\left(\sum_{i=2}^{\frac{n/k-1}{2}} \norm{z^{2i}+z^{2i+1}}{2}\right)\\
&\ge (1-\epsilon) \norm{z^1}{2} - \frac{(1+\epsilon)}{\sqrt{2k}} \norm{z^1}{1} \ge (1-\epsilon) \norm{z^1}{2} - \frac{(1+\epsilon)}{\sqrt{2}} \norm{z^1}{2}\\
&\ge 0.1 \norm{z^1}{2}
\end{align*}
for small enough $\epsilon$.  Next, note that
$$
\norm{z^1}{2} \ge \frac{\norm{z^1}{1}}{\sqrt{k}} \ge \frac{\norm{z^2 + \dots + z^{n/k}}{1}}{\sqrt{k}} = \frac{\sum_{i=2}^{n/k}\norm{z^i}{1}}{\sqrt{k}} \ge \sum_{i=3}^{n/k}\norm{z^i}{2} \ge \left\| \sum_{i=3}^{n/k} z^i \right\|_2
$$
Hence we have $3\norm{z^1}{2} \ge \norm{z}{2}$, and finally we have $\norm{Az}{2} \gtrsim \|z\|_2$.

\subsection{Recover a non-Sparse Vector}
In Theorem~\ref{cs-l1}, we have shown how to recover a sparse $x$.  Now, what if $x$ is not sparse?  It turns out that we may use the similar technique.  Denote $x_a$ as the vector that zero out all but $a$ largest items from $x$.  We leave it as an exercise that showing the following property that suggests, essentially, we can solve the problem assuming that $x$ is a sparse vector:
\begin{exercise}
	If $A$ is a RIP matrix, then
	$$ \norm{A(x-x_{2k})}{2} \le \frac{\norm{x-x_k}{1}}{\sqrt{k}}(1+\epsilon) $$
\end{exercise}
This property suggests that we may use the technique is the previous lectures to get $\hat x$ from $y=Ax$ with $l2/l1$ guarantee $$\norm{\hat x-x}{2} \lesssim \frac{1}{\sqrt{k}}\norm{x-x_k}{1} $$
And after further sparsifying, we can get a solution $\hat x^*$ that satisfies the $l1/l1$ guarantee:
$$\norm{\hat x^*-x}{1} \lesssim \norm{x-x_k}{1} $$

\section{Comparison}
In this section we provide a quick reference to the comparison between different recovery methods and their properties within the following table. The Count-min and Count-sketch results are based on the previous lectures.\\
\begin{table}[!hbp]
\centering
\begin{tabular}{|c|c|c|c|}
\hline
lable  & $L^1$ minimization & Count-min & Count Sketch \\
\hline
Recovery time & $n^3$ & \multicolumn{2}{|c|}{$n\log n$} \\
\hline
Space & $k \log\frac{n}{k} $ & \multicolumn{2}{|c|}{$k\log n$} \\
\hline
Guarantee & \textit{$l_2/l_1$} &  \textit{$l_\infty/l_1 \implies l_1/l_1$} & \textit{$l_\infty/l_2 \implies l_2/l_2$} \\
\hline
Notes & If $A$ satisfy RIP, works for all $x$ &  \multicolumn{2}{|c|}{For each $x$, $A$ works with high probability}\\
\hline
\end{tabular}
\caption{table of comparison}
\end{table}\\
In class we also discussed a related theorem without a detailed proof. The theorem is stated as below and the proof will be given in the next lecture if possible.
\begin{theorem}
	We cannot get an algorithm with deterministic $l_2/l_2$ guarantee with $m=o(n)$.
\end{theorem}



\bibliographystyle{alpha}

%\begin{thebibliography}{42}
%
%\bibitem[CRT06]{CRT06}
%Candes, E. J., Romberg, J. K., Tao, T.
%\newblock Stable signal recovery from incomplete and inaccurate measurements.
%\newblock {\em Communications on Pure and Applied Mathematics}, 59(8), 1207-1223.
%
%\bibitem[BCDH10]{BCDH10}
%R. Baraniuk, V. Cevher, M. F. Duarte, C. Hegde.
%\newblock Model-based compressive sensing.
%\newblock {\em IEEE Transactions on Information Theory}, 59(8), vol. 56, num. 4, p. 1982-2001, 2010.
%
%\end{thebibliography}

\end{document}


