\documentclass[11pt]{article}
\usepackage{amsmath,amssymb,amsthm}
\usepackage{graphicx}

\DeclareMathOperator*{\E}{\mathbb{E}}
\let\Pr\relax
\DeclareMathOperator*{\Pr}{\mathbb{P}}

\newcommand{\eps}{\epsilon}
\newcommand{\inprod}[1]{\left\langle #1 \right\rangle}
\newcommand{\R}{\mathbb{R}}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {\bf CS 395T: Sublinear Algorithms } \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[4]{\handout{#1}{#2}{#3}{Scribe: #4}{Lecture #1}}

\newtheorem{theorem}{Theorem}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{observation}[theorem]{Observation}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{assumption}[theorem]{Assumption}

% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex

\begin{document}

\lecture{15 --- Oct 16, 2014}{Fall 2014}{Prof.\ Eric Price}{Chi-Kit Lam}

\section{Overview}

In this lecture we will talk about adaptive sparse recovery.

\section{Adaptivity in group testing}
In sparse recovery, we have
\begin{equation*}
y = A x =
\begin{pmatrix}
\inprod{v_1, x} \\
\inprod{v_2, x} \\
\vdots \\
\inprod{v_m, x} \\
\end{pmatrix} .
\end{equation*}
In our previous non-adaptive setting, $v_i$'s were chosen independently.
Intuitively, it seems we may be able to do better if they are not
independent. So, in the adaptive setting we talk about today, $v_i$ is
chosen dependent on
$\inprod{v_1, x}, \inprod{v_2, x}, \ldots, \inprod{v_{i-1}, x}$.
The idea is, for some architecture like the single-pixel camera,
linear measurements are being taken sequentially. So, you can
choose what measurement to take based on what you have found previously.

\subsection{Non-adaptive group testing}
Consider the example of blood testing, where you have $n$ people and
some small fraction of them are carrying disease. You want to know which of
them has the disease. This problem was first studied in World War II to test
syphilis in the army. Group testing is also used by the Orthodox Jewish
community for pre-dating genetic testing.

The trivial solution is to test every single person. Alternatively,
to make it faster, we can mix together blood samples. For example,
you can mix together the blood samples of 10 people, and test whether
any of these 10 people got the disease. If the chance of the disease is
$1/1000$, you can make your poll size $\approx 1000$. Then the chance
of this pool has a positive or negative is $\approx 1/2$. So, you can get
$\approx 1$ bit of information per test (either yes or no). And if you do
$\approx k \log n$ such trial, using a union bound, with high probability,
you can find out which person has the disease.

Let $x \in \{0,1\}^n$ be the indicator vector of the set with
disease. Let $v \in \{0,1\}^n$ be the indicator vector of the set that you mix
together. Then, each test observes whether $\inprod{v,x} \neq 0$. If
$x$ is $k$-sparse, we can choose $m = O (k \log n)$ random $v$'s, where each
$v$ has $n/k$ ones and each person is in $d = O (\log n)$ tests. We can
thinking of the design of the group testing as a bipartite graph similiar
to Figure~\ref{fig:graph}.

\begin{figure}[h]
\centering
\includegraphics{fig_graph}
\caption{Bipartite graph corresponding to the design of group testing.
The $n$ left vertices correspond to the $n$ people and the $m$ right
vertices correspond to the $m$ tests. \label{fig:graph} }
\end{figure}

For each person, suppose their tests are $t_1, \ldots, t_d$.
If that person has disease, then all tests $t_i$ are positive.
Otherwise,
\begin{align*}
\Pr [ \text{test } t_i \text{ is positive} ]
= & \Pr [ \text{some sample } x_j \text{ in test } t_i \text{ is non-zero} ] \\
= & 1 - \Pr [ \text{all samples } x_j \text{ in test } t_i \text{ is zero} ] \\
\approx & 1 - (1-k/n)^{n/k} \approx 1 - 1/e .
\end{align*}
So,
\begin{equation*}
\Pr [\text{all tests } t_i \approx (1-1/e)^d \leq 1/n^{10}
\end{equation*}
for some $d = O(\log n)$.

Therefore, non-adaptive random test uses $O(k \log n)$ measurements,
and for all such $x$, with high probability it returns the correct answer.

\subsection{Adaptive group testing}
In most cases, you can actually tell whether you are certain about the result.
Consider the example in Figure~\ref{fig:example}. We know for certain that people
$2$, $4$ and $6$ do not carry disease, but people $1$ and $6$ do. The
result for person $3$ is ambiguous. In general, we can first cross out
all people who are in at least one negative test. For the remainng people,
if they are the only remaining participant of some positive test,
then we are certain that they carry disease. Otherwise, they are ambiguous.
\begin{figure}[h]
\centering
\includegraphics{fig_example}
\caption{Example of ambiguous group testing result. The left vertices
correspond to $6$ people and the right vertices correspond to $6$ tests
labeled by the test results. \label{fig:example} }
\end{figure}

We have shown that for all such $x$, the number of ambiguity is $0$
with probability $1 - n^{- \Omega(1)}$. In fact, we can show that
for all such $x$, the number of ambiguity is less
than $t$ with probability $1 - n^{- \Omega (t)}$. Take $t = O(k)$ and
apply union bound over $\binom{n}{k}$ possible values
of $x$. We can show that with high probability, for all such $x$,
the number of ambiguity is less than $O(k)$.
This uniform guarantee is analogous to the result that a Gaussian matrix
with high probability satisfies RIP, but it does not give us an explicit
construction.

So, after one round, with high probability, for all such $x$, there
are only $O(k)$ ambiguity. This means that in two rounds, it can eliminate
all ambiguity uniformly with $O (k \log n)$ tests. However, with one round,
uniformity requires $\Omega (k^2 \log_k n)$ tests,
and $O (k^2 \log n)$ tests suffice. Therefore, adaptivity helps group testing.

\section{Adaptivity in sparse recovery}
In group testing, we get at most $1$ bit of information per measurement,
so we need $O (k \log \frac{n}{k})$ measurements to distinguish all
$\binom{n}{k}$ possible inputs. The difference of sparse recovery
and group testing is that we do not just
learn whether $\inprod{v,x} = 0$ or not, but also the value of
$\inprod{v,x}$. In the case of real measurement, we can in principle get
a lot of information per measurement. However, if we need to tolerate noise,
we can show that we can only get $O(1)$ effective bit of information
per measurement.
So, the question is whether adaptivity can help in sparse recovery.

The result is that adaptivity provides no improvement in the uniform setting.
In particular, for $\ell_2 / \ell_2$ sparse recovery,
$\Omega( \frac{k}{\eps} \log \frac{n}{k})$
measurements are necessary \cite{FR13}.

In contrast, adaptivity can help in the non-uniform setting.
It has been shown that adaptive $\ell_2 / \ell_2$ sparse recovery in the
non-uniform setting is possible with
$O (\frac{k}{\eps} + k \log \log \frac{n}{k})$ measurements \cite{IPW11}
and $\Omega (\frac{k}{\eps} + \log \log n)$ measurements are required \cite{ACD13,PW13}.

\subsection{Lower bound of non-adaptive sparse recovery}
Consider $k=1$ and $\eps = 1$. We will prove a lower bound of $m = \Omega(\log n)$
measurements for non-adaptive sparse recovery in the non-uniform setting.

Suppose $A \sim \mathcal{P}_A$ such that for all $x$, it works
with high probability. Let $\mathcal{P}_x$ be a distribution of $x$. Then,
\begin{align*}
& \forall x \Pr_{A \sim \mathcal{P}_A} [ \text{correct} ] > 1 - n^{-10} \\
\implies & \Pr_{x \sim \mathcal{P}_x, A \sim \mathcal{P}_A}
[ \text{correct} ] > 1 - n^{-10} \\
\implies & \exists A \Pr_{x \sim \mathcal{P}_x}
[ \text{correct} ] > 1 - n^{-10} .
\end{align*}
This is known as Yao's minimax principle. Using this, it suffices
to exhibit some distribution $\mathcal{P}_x$ such that for all fixed
deterministic $A$, if the $\Pr_{x \sim \mathcal{P}_x}
[ \text{correct} ] > 1 - n^{-10}$, then $m \geq \Omega (\log n)$.

Consider $x = e_j + w$, where $e_j$ has a one in a random position $j$ and
$w$ is drawn according to $\mathcal{N} (0, I/100n)$. Since $\|w\|_2 \approx
0.1$, $\ell_2 / \ell_2$ recovery gets $x$ to $\pm 0.2$. So, rounding $x$
recovers the value of $j$, which contains $\log n$ bits of information.
Therefore, the communication
\begin{equation*}
j \to x \to Ax \to \hat{x} \to \hat{j} = j
\end{equation*}
must cost $\Omega(\log n)$ bits. If we can show that each measurement
$\inprod{v,x}$ contains information $I(\inprod{v, e_j}; \inprod{v,x})
\lesssim 1$, then by Lemma \ref{lem:subadditivity} which we will argue,
we can show that $\log n = I(j; A x) \leq I (A e_j; A x) \lesssim m$.

Observe that each measurement is $\inprod{v,x} = \inprod{v, e_j}
+ \frac{\|v\|_2}{10 \sqrt{n}} \mathcal{N} (0,I)$. By Shannon-Hartley
Theorem, $I(\inprod{v, e_j}, \inprod{v,x}) \leq 0.5 \log (1 + \mathrm{SNR})$,
where the signal-to-noise ratio in this case equals
\begin{equation*}
\mathrm{SNR} = \frac{\E[\mathrm{signal}^2]}{\E[\mathrm{noise}^2]}
= \frac{\sum_j \inprod{v, e_j}^2 / n}{\|v\|_2^2/100n}
= \frac{\|v\|_2^2 / n}{\|v\|_2^2/100n}
= 100 .
\end{equation*}
So, $I(\inprod{v, e_j}, \inprod{v,x}) \lesssim 1$. Notice that
non-adaptivity is required for $\E[\mathrm{signal}^2]$ above.

\subsection{Entropy and mutual information}
For a discrete random variable $x \sim p$, the entropy of $x$ defined as
\begin{equation*}
H (x) = \sum_i p_i \log \frac{1}{p_i} .
\end{equation*}
is the expected/asymptotic description length. If $x$ is continuous,
we write $h(p)$ and the summation becomes integration.

For example, if
\begin{equation*}
x =
\begin{cases}
0 & \text{with probability $999/1000$} \\
1 & \text{with probability $1/1000$}
\end{cases},
\end{equation*}
then
\begin{equation*}
H (x) = \frac{999}{1000} \log \frac{1000}{999} + \frac{1}{1000} \log 1000
\approx \frac{999}{1000} 0.001 + \frac{1}{1000} \log 1000
\approx \frac{\log 1000}{1000} .
\end{equation*}

The mutual information between $a$ and $b$ is defined as
\begin{equation*}
I(a;b) = H(a) - H(a,b).
\end{equation*}

\subsection{Subadditivity of mutual information}
In general, mutual information is not subadditive. For example,
consider $x$ and $b$ drawn independently and uniformly from $\{0,1\}$.
Let $y_1 = x \oplus b$ and $y_2 = b$. Then, $I(x; y_1) = I(x; y_2) = 0$
but $I(x; (y_1, y_2)) = 1$.

We can assume matrix $A$ has orthonormal rows. (If $A = U \Sigma V^\text{T}$
is the SVD factorization of $A$, then $\Sigma^{-1} U^\text{T} A$ has
orthonormal rows and this is an invertible transformation.)

\begin{lemma}
\label{lem:subadditivity}
$I(A e_j; A x) \leq \sum_i I(\inprod{v_i, e_j}; \inprod{v_i, x})$.
\begin{proof}
Consider $A x = A e_j + A w$. Since $w \sim \mathcal{N} (0, I/100n)$
and $A$ has orthonormal rows, entries in $A w$ are independent $A e_i$
and each other. So, it suffices to prove the following lemma.
\end{proof}
\end{lemma}

\begin{lemma}
If $y = \overline{y} + w \in \R^n$ such that $w_i$'s are independent
of $\overline{y}$ and each other, then $I(\overline{y}; y) \leq \sum_i
I (\bar{y}_i; y_i)$.
\begin{proof}
\begin{align*}
I(\overline{y}; y) = & h(y) - h(y | \overline{y}) \\
= & h(y) - h(w) \\
= & \sum_i h(y_i | y_1, \ldots, y_{i-1})
  - \sum_i h(w_i | w_1, \ldots, w_{i-1}) \\
= & \sum_i h(y_i | y_1, \ldots, y_{i-1}) - \sum_i h(w_i) \\
\leq & \sum_i h(y_i) - \sum_i h(w_i) \\
= & \sum_i h(y_i) - \sum_i h(y_i | \overline{y}_i) \\
= & \sum_i I(\overline{y}_i; y_i)
\end{align*}
\end{proof}
\end{lemma}

\bibliographystyle{alpha}

\begin{thebibliography}{42}

\bibitem[ACD13]{ACD13}
Ery~Arias-Castro, Emmanuel~J.~Cand\`{e}s, Mark~A.~Davenport.
\newblock On the Fundamental Limits of Adaptive Sensing.
\newblock {\em IEEE Transaction on Information Theory} 59(1): 472-481 (2013)

\bibitem[FR13]{FR13}
Simon~Foucart, Holger~Rauhut.
\newblock A Mathematical Introduction to Compressive Sensing.
\newblock Applied and Numerical Harmonic Analysis, Birkhaüser. 

\bibitem[PW13]{PW13}
Eric~Price, David~P.~Woodruff.
\newblock Lower Bounds for Adaptive Sparse Recovery.
\newblock {\em SODA} 2013: 652-663.

\bibitem[IPW11]{IPW11}
Piotr~Indyk, Eric~Price, David~P.~Woodruff.
\newblock On the Power of Adaptivity in Sparse Recovery.
\newblock {\em FOCS} 2011: 285-294.

\end{thebibliography}

\end{document}