\documentclass[11pt]{article}
\usepackage{amsmath,amssymb,amsthm,bbm}
\usepackage{tkz-berge}
\usepackage{tikz,float}
\usepackage{pgfplots}

\DeclareMathOperator*{\E}{\mathbb{E}}
\let\Pr\relax
\DeclareMathOperator*{\Pr}{\mathbb{P}}
\DeclareMathOperator*{\Vol}{\mathsf{Vol}}


\newcommand{\eps}{e}
\newcommand{\inprod}[1]{\left\langle #1 \right\rangle}
\newcommand{\R}{\mathbb{R}}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {\bf CS 395T: Sublinear Algorithms } \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[4]{\handout{#1}{#2}{#3}{Scribe: #4}{Lecture #1}}

\newtheorem{theorem}{Theorem}[subsection]
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{observation}[theorem]{Observation}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{property}[theorem]{Property}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{rules}[theorem]{Rule}
\newtheorem{assumption}[theorem]{Assumption}

% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex

\begin{document}

\lecture{12 --- October 4, 2016}{Fall 2016}{Prof.\ Eric Price}{Akshay D Kamath, Surbhi Goel}

\section{Lower Bounds for $p^{th}$ Moment $(p>2)$}
We will prove lower bounds for $p^{th}$ Moment $(p>2)$ using information theory. Suppose we have an algorithm $Alg$ and a distribution on $A\in \mathbb{R}^{m \times n}$ such that given $x \in \mathbb{R}^n$, running $Alg$ on $(Ax)$ outputs $v$ that satisfies $v = (1 \pm \epsilon)||x||_p^p$ w.p. $1 - \delta$.

Consider the following setup,
\begin{itemize}
\item Alice has bit $b \in \{0,1\}$ drawn uniformly at random. She chooses $x^*,w \in \mathbb{R}^n$ such that $w \sim N(0, \sigma^2 I_n)$ and
\begin{equation*}
x^* \; = \; \begin{cases}
0& \text{if~} b = 0\,,\\[3pt]
e_i & \text{w.p.~} 1/n\ \forall i \in \{1, \ldots, n\} \text{~if~} b=1 \,.\\[3pt]
\end{cases}
\end{equation*}
She sends $A(x^* + w)$ to Bob.
\item Bob runs $Alg$ on $A(x^* + w)$ to estimate $||x^* + w||_p^p$ and outputs $\hat{b} = Alg(A(x^*+w)) > 1.5$.
\end{itemize}

We can easily see that,
\begin{equation*}
||x^* + w||_p^p \; \approx \; \begin{cases}
n \sigma^p& \text{if~} b = 0\,,\\[3pt]
1 + n \sigma^p & \text{if~} b=1 \,.\\[3pt]
\end{cases}
\end{equation*}

\begin{claim}
$P_{err} = Pr[\hat{b} \neq b] < 1/3$.
\end{claim}
\begin{proof}
Set $\sigma < n^{-1/p}$, then getting $\epsilon = 0.1$ approximation to $||x*+w||_p^p$ lets Bob learn $b$ with error probability $\delta + o(1) < 1/3$ where $\delta$ is the error made in streaming and $o(1)$ is the chance $||x^* + w||_p^p$ deviates by a constant. Thus we will get $P_{err} = Pr[\hat{b} \neq b] < 1/3$.
\end{proof}

\begin{claim}
Information between $\hat{b}$ and $b$ is $\leq \frac{m}{n^{1 - 2/p}}$.
\end{claim}
\begin{proof}
WLOG, we can assume that rows of $A$ are orthonormal. To see this, take SVD of $A = U \Sigma V^T$. Then, if algorithm works using $A$, it also works using $V^T$. Given $V^T$, we can construct $Alg'$ that runs on $V^Tx$ as $Alg'(V^Tx) = Alg(U \Sigma V^T x)$. Also note that $V^T has \leq m$ rows.

Let $v$ be a row of $A$, we have
\[
I(\langle v, x^* + w \rangle, \langle v, x^* \rangle) = I(\langle v, x^* \rangle + \langle v, w \rangle, \langle v, x^* \rangle)
\]
Since $w$ is drawn from a Gaussian distribution, $\langle v, w \rangle$ is additive white Gaussian noise (AWGN). Thus, using Shanon-Hertley, we get
\[
I(\langle v, x^* + w \rangle, \langle v, x^* \rangle) \leq \frac{1}{2} \log \left( 1 + \frac{E[\langle v, x^* \rangle^2]}{E[\langle v, w \rangle]^2} \right) = \frac{1}{2} \log \left( 1 + \frac{1/2n}{\sigma^2} \right) \leq \frac{1}{4n\sigma^2} = \frac{1}{4n^{1 - 2/p}}
\]
since
\begin{align*}
E[\langle v, x^* \rangle^2] = \frac{1}{2} \sum_{i=1}^n v_i^2 \cdot \frac{1}{n} = \frac{1}{2n}
&  &E[\langle v, w \rangle^2] = \sum_{i=1}^n v_i^2 \sigma^2 = \sigma^2
\end{align*}
Note that this only holds for single measurement, we could potentially get more information from multiple observations even if we do not get from one. We will show that information can be bounded additively in this case.
\begin{lemma}
If $a_i = b_i + w_i\ \forall i \in [n]$ such that each $w_i$ is independent of each other and of $b_i$, then
\[
I(a,b) \leq \sum_{i=1}^n I(a_i, b_i)
\]
\end{lemma}
\begin{proof}
We have,
\begin{align*}
I(a,b) 	&= h(a) - h(a|b) = h(a) - h(a-b|b) = h(a) - h(w|b) = h(a) - h(w) \\
        &\leq \sum_{i=1}^n h(a_i) - h(w_i) = \sum_{i=1}^n h(a_i) - h(w_i|b_i) = \sum_{i=1}^n h(a_i) - h(a_i - b_i|b_i) \\
        &= \sum_{i=1}^n h(a_i) - h(a_i|b_i) =\sum_{i=1}^n I(a_i,b_i)
\end{align*}
Hence, proved.
\end{proof}
Using the above lemma, we get
\[
I(A(x^* + w), Ax^*) \leq \frac{m}{4n^{1 - 2/p}}
\]
Since $b \rightarrow x^* \rightarrow Ax^* \rightarrow A(x^* + w) \rightarrow \hat{b}$, 
\[
I(b, \hat{b}) \leq \frac{m}{4n^{1 - 2/p}}.
\]
Hence, proved.
\end{proof}
Using the above claim, we get
\[
h(b|\hat{b}) \geq h(b) - \frac{m}{4n^{1 - 2/p}} = 1 - \frac{m}{4n^{1 - 2/p}}
\]
This shows that even if you know $\hat{b}$, there is sufficient entropy left in $b$ hence still sufficient chance of error. Now, if $m < n^{1 - 2/p}$, we get $h(b|\hat{b}) \geq 3/4$ and
\[
h(b \neq \hat{b}) \geq h(b \neq \hat{b}|\hat{b}) \geq h(b|\hat{b}) \geq 3/4
\]
\begin{figure}[H]
\centering
\begin{tikzpicture}
\begin{axis}[legend pos = north west,xmin=0,xmax=1,ymin=0,ymax=1,xlabel=$Pr(x)$,ylabel=$h(x)$]
	\addplot[domain=0:1, blue, ultra thick, samples=500] {(-x*ln(x) - (1-x)*ln(1-x))/ln(2)};
\end{axis}
\end{tikzpicture}
\caption{Entropy Curve for bit $x$}
\end{figure}
From the entropy curve, it is easy to see that this gives us a contradiction since we have $Pr_{err} = Pr[\hat{b} \neq b] <1/3$ by the first claim. 
% Surbhi: This doesn't seem to work out for these constants!


\section{Packing and Covering Numbers}

In this section we discuss packing and covering numbers. We will use results shown here to prove results on compressed sensing in a later lecture.

\subsection{Definitions}
\begin{definition}
A metric space is an order pair $(X,d)$ where $X$ is a set and $d:X\rightarrow \mathbb{R}$ is a distance metric with the following properties for all $x,y\in X$:
\begin{enumerate}
\item[(i)]  $d(x,y)\geq 0$
\item[(ii)] $d(x,y)=0\Leftrightarrow x=y$
\item[(iii)] $d(x,y)=d(y,x)$
\item[(iv)] $\forall z\in X,  d(x,y)\leq d(x,z) + d(z,y)$   
\end{enumerate}

\end{definition}

\begin{definition}
An $\epsilon$-cover of $X$ with respect to $d$ is a collection of points $\{x_1, x_2, \cdots x_n\}\subseteq X$ such that $\forall y\in X \min\limits_{i\in[n]} d(y,x_i)\leq \epsilon$ 
\end{definition}

\begin{definition}
The covering number $N(\epsilon, X,d)$ is the minimum $n$ such that there exists an $\epsilon$-cover of size $n$.
\end{definition}

\begin{definition}
The metric entropy of $\log(N(\epsilon, X,d))$. 
\end{definition}

We may look at a few examples to understand these definitions.\\
\textbf{Example 1:} Consider $X=[-1,1]$ under the distance metric $d(x,y)=|x-y|$. The set $\{0, \pm 2\epsilon, 4\epsilon, \cdots \}\cap [-1,1]$ is an $\epsilon$-cover. So, we know that $N(\epsilon, X, d)\leq \frac{2}{2\epsilon} +1$.\\

\textbf{Example 2:} Consider $X=[-1,1]^n$ under the distance metric $d(x,y)=\| x- y\|_\infty$. We may view this as covering the box $[-1,1]^n$ using $\epsilon^n$ sized  
boxes. So, we need at most $(\frac{1}{\epsilon}+1)^n$ boxes. $N(\epsilon, X, d)\leq (\frac{1}{\epsilon}+1)^n$.\\

\begin{definition}
An $\epsilon$-packing of $X$ with respect to $d$ is a collection of points $\{x_1, x_2, \cdots x_n\}\subseteq X$ such that $\min\limits_{i\neq j\in[n]} d(x_j,x_i)\geq \epsilon$ 
\end{definition}

\begin{definition}
The packing number $M(\epsilon, X,d)$ is the maximum $n$ such that there exists an $\epsilon$-packing of size $n$.
\end{definition}


In Homework 3, we will prove the following:
\begin{lemma}\label{HW3.1}
$M(2\epsilon, X, d) \leq N(\epsilon, X,d)\leq M(\epsilon, X, d)$

\end{lemma}

\subsection{Bounds on Covering Number}
In this section we study bounds on the covering number of the unit $L_q$-ball under $L_p$ distance. We wish to bound $N(\epsilon, B_q^d, \|\cdot\|_p)$ where $\| x \|_p = (\sum x_i^p)^\frac{1}{p}$ and $ B_q^d = \{x\in \mathbb{R}^d\big| \|x\|_p\leq 1\}$.

We know that if $p\geq q$, then we have $\|x\|_p\leq \|x\|_q$. This means $p\geq q \Rightarrow B_q^d\subseteq B_p^d$.

Let us use $\Vol(S)$ to denote the volume of the set $S$. We note that in $d-$dimensions, $\Vol(aS)=a^d\Vol(S)$

\begin{theorem}
$\frac{1}{\epsilon^d} \frac{\Vol(B_q^d)}{\Vol(B^d_p)}\leq N(\epsilon, B_q^d, \|\cdot\|_p) \leq (\frac{2}{\epsilon})^d \frac{\Vol(B_q^d+\frac{\epsilon}{2}B_p^d)}{\Vol(B^d_p)}$
\end{theorem}

\begin{proof}
\textbf{Lower Bound:}\\
Let $x_1, \cdots, x_n$ be an $\epsilon$-cover of $B_q^d$ under the $L_p$ norm.\\
Then, 
$$B_q^d \subseteq \bigcup\limits_{i\in [n]} (x_i + \epsilon(B_p^d))$$

So, we may bound the volume:
\begin{align*}
\Vol(B_q^d) &\leq n\cdot\Vol(\epsilon B_p^d)\\
	&=n\cdot \epsilon^d \cdot \Vol(B_p^d)
\end{align*}

So, $n=N(\epsilon, B_q^d, \|\cdot\|_p)\geq \frac{1}{\epsilon^d} \frac{\Vol(B_q^d)}{\Vol(B^d_p)}$

\textbf{Upper Bound:}\\

Let $x_1, \cdots , x_m$ be an $\epsilon$-packing of $B_q^d$. 

This means that the balls $x_i+\frac{\epsilon}{2} B_p^d$ are disjoint (except at the surface). We also know that all of these balls are contained in $B_q^d+\frac{\epsilon}{2}B_p^d$ since some of these balls might be on the surface of $B_q^d$.

So, 
 \begin{align*}
\Vol(B_q^d + \frac{\epsilon}{2}B_p^d) &\geq m\cdot\Vol(\frac{\epsilon}{2} B_p^d)\\
	&=m\cdot (\frac{\epsilon}{2})^d \cdot \Vol(B_p^d)
\end{align*}

So, we conclude that $m=M(\epsilon, B_q^d, \|\cdot\|_p)\leq (\frac{2}{\epsilon})^d \cdot \frac{\Vol(B_q^d + \frac{\epsilon}{2}B_p^d)}{\Vol(B_p^d)}$

Using Lemma \ref{HW3.1}, we may conclude that $N(\epsilon, B_q^d, \|\cdot\|_p)\leq (\frac{2}{\epsilon})^d \cdot \frac{\Vol(B_q^d + \frac{\epsilon}{2}B_p^d)}{\Vol(B_p^d)}$

\end{proof}

Let us examine this Theorem under the values $p=2$ and $q=1$.

$\Vol(B_1^d) = \frac{2^d}{d!}$ and $\Vol(B_2^d)=\frac{\pi^\frac{d}{2}}{(d/2)!}$ when $d$ is even. 

The bound on $N=N(\epsilon, B_1^d, \|\cdot\|_2)$ which we get by applying this theorem can be written in terms of metric entropy as:
$$ d\log(1/\epsilon)-\frac{d}{2}\log(d)\leq \log(N)\leq d\log(\frac{2}{\epsilon})$$ 

Observe that this gives us a tight bound when $\epsilon<<\frac{1}{d}$. However, when $\epsilon>>\frac{1}{\sqrt{d}}$, the bound is trivial.

Intuitively, we may explain this as follows: The $L_1$ ball in $d-$dimensions should be viewed as being ``spikey''. The $\epsilon B_2^d$ balls, which we use to cover the tips of these spikes incur a huge loss in volume i.e a large volume which we account for in the bound is outside the volume which we wish to cover. 

We will use a different method in order to prove that $\log(N)\leq \frac{1}{\epsilon^2} \log(d)$ in the next lecture.


\end{document}