\documentclass[11pt]{article}
\usepackage{amsmath,amssymb,amsthm}

\DeclareMathOperator*{\E}{\mathbb{E}}
\let\Pr\relax
\DeclareMathOperator*{\Pr}{\mathbb{P}}
\newcommand{\norm}[1]{\left\lVert#1\right\rVert}
\newcommand{\eps}{\epsilon}
\newcommand{\inprod}[1]{\left\langle #1 \right\rangle}
\newcommand{\R}{\mathbb{R}}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {\bf CS 395T: Sublinear Algorithms } \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[4]{\handout{#1}{#2}{#3}{Scribe: #4}{Lecture #1}}

\newtheorem{theorem}{Theorem}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{observation}[theorem]{Observation}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{assumption}[theorem]{Assumption}

% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex

\begin{document}

\lecture{11 --- 29th September, 2016}{Fall 2016}{Prof.\ Eric Price}{Vatsal Shah, Yanyao Shen}

\section{Overview}
In previous lectures, we have shown how to estimate $l_2$ norm using AMS-sketch and how to estimate number of distinct elements. As a result, we get $O(\frac{1}{\epsilon^2})$ for $l_2$ and $O(\log^c n)$ for $l_0$ respectively.

In this lecture, we will estimate $p$th moment, where $p\in (0,2)$ and show that it is doable in $O(\log^c n)$ space. On the other hand, when $p>2$, linear sketches require $\Omega(n^{1-2/p})$ words, and is not doable in $poly\log n$ space.

\section{A special case when $p=1$}

First, recall the algorithm for $p=2$.
\begin{itemize}
\item Select $v\in \mathbb{R}^n$, where $v_i\sim N(0,1)$, then $\left \langle v,x \right \rangle \sim N(0, \|x\|^2)$
\item Take $\left \langle v_1,x \right \rangle , \left \langle v_2,x \right \rangle, \left \langle v_3,x \right \rangle,\cdots$: samples at $N(0,\|x\|^2)$ and then estimate. This requires $O(\frac{1}{\epsilon^2}\log \frac{1}{\delta})$ samples.
\end{itemize}

For $p=1$, instead of choosing samples from Gaussian distribution, we use Cauchy distribution.


\subsection{Cauchy Distribution Basics}
In this section, we will look at the definitions and properties of Cauchy distribution:
\begin{itemize}
\item Standard form of Cauchy Distribution:
\begin{equation}
p(x) = \dfrac{1}{\pi} \dfrac{1}{x^2 + 1}
\end{equation}
\item General Form: Cauchy Distribution with scale factor $\gamma$
\begin{equation}
p(x) = \dfrac{1}{\pi \gamma} \dfrac{1}{\left(\frac{x}{\gamma}\right)^2 + 1}
\end{equation}

\item \textbf{Claim:} If $X_1 \sim Cauchy(\gamma_1)$, $X_2 \sim Cauchy(\gamma_2)$, then $X_1 + X_2 \sim Cauchy(\gamma_1 + \gamma_2)$ 

\textbf{Proof:}
The Fourier Transform of standard Cauchy Distribution is given as follows:
\begin{align*}
\mathcal{F}_x(t) &= \mathbb{E}[cos(2\pi t x)] \\
&= \int_{-\infty}^{\infty} \exp(2\pi i t x) \dfrac{1}{\pi(1 + x^2)} dx \\
&= \exp(-2 \pi |t|)
\end{align*}

Inverse Fourier Transform of standard Cauchy Distribution is:
\begin{align*}
\mathcal{F}_x^{-1}(t) &= \int_{-\infty}^{\infty} \exp(2\pi i t x) \exp(2\pi |t|)  dx \\
&= \int_{-\infty}^{0} \exp(2\pi i t x - 2 \pi t) dt + \int_{0}^{\infty} \exp(2\pi i t x + 2 \pi t) dt \\
&= \dfrac{1}{2\pi(1-ix)} + \dfrac{1}{2\pi(1+ix)}\\
&= \dfrac{1}{\pi(1 + x^2)} = p(x)
\end{align*}

We can similarly show that the Fourier transform of $\mathcal{F}_{Cauchy(\gamma)}(t)$ is $\exp(-2\pi|t|\gamma)$.

Thus, if $X_1 \sim Cauchy(\gamma_1)$, $X_2 \sim Cauchy(\gamma_2)$, then:
\begin{align*}
\mathcal{F}_{X_1+X_2}(t) &= \mathcal{F}_{X_1}(t)\mathcal{F}_{X_2}(t) = \exp\left(-2\pi|t|(\gamma_1 + \gamma_2) \right)\\
&= \mathcal{F}_{Cauchy(\gamma_1 + \gamma_2)}(t)
\end{align*}

Now, if we have $X_1, X_2, \dots, X_n \sim Cauchy(1)$, then $\dfrac{X_1 + X_2 + \dots X_n}{n} \sim Cauchy(1)$.

This seemingly violates the law of large numbers which states that the distribution of the average of $n$ random variables approaches that of a Gaussian as $n$ increases. The law of large numbers is only applicable for random variables that have a finite expectation which does not hold for Cauchy random variables.

\item \textbf{$p-stable$ distribution}:\\
Let $p>0$ be a real number. A probability distribution $\mathcal{D}$ is a $p-stable$ distribution if $\forall a_1, a_2, \dots a_n$ and  $X_1, \dots X_n ~ \mathcal{D}$ are independently chosen, $\sum_i a_i X_i$, $\bar{a} X$ have the same distribution and $X \sim D $ and $\bar{a} = \norm{a}_p$

$p-stable$ distribution are typically defined $0<p<2$ and do not exist for $p > 2$

Using the above definition, we can easily show that Cauchy is a $1-$stable distribution while Gaussian is a $2-$stable distribution.

\end{itemize}


\subsection{Algorithm for $p=1$}
Here is the algorithm.
\begin{itemize}
\item For $i = 1,\cdots, m$, sample $A_{i,:}$'s elements from $Cauchy(\gamma=1)$ distribution.
\item Then, $y_i = A_{i,:}x$ follows distribution $Cauchy(\gamma = \|x\|_1)$, according to properties in section 2.1.
\item Store $y_1, \cdots, y_m$.
\item Let the median of $|y_i|$ be our estimator for $\|x\|_1$.
\end{itemize}

\paragraph{Question:}
Given $m$ samples of Cauchy with unknown $\gamma$, how well can we estimate $\gamma$?

Use median. Then, how does median of $|y_i|, i\in [n]$ behave?

\paragraph{Claim:}
the median of $|y_i|, i\in [n]$ $\rightarrow c\gamma$.
\begin{itemize}
\item How do we know $c$?
\item How fast?
\end{itemize}
To answer the first question, we can compute $c$ directly. $c$ should satisfy:
\begin{align*}
\Pr(|y_i|<c\gamma) &= \frac{1}{2}\\
\Rightarrow 2\int_{0}^{c\gamma} \frac{1}{\pi\gamma ((\frac{x}{\gamma})^2+1)} dx & = \frac{1}{2} \\
\Rightarrow 2\int_{0}^{c} \frac{1}{\pi(x^2+1)}dx &= \frac{1}{2}\\
\Rightarrow \frac{2}{\pi} \arctan x|_{0}^c &= \frac{1}{2} \\
\Rightarrow \arctan c & = \frac{\pi}{4} \Rightarrow c = 1.
\end{align*}

To answer the second question, we use the same methodology as in Lecture 7. Notice that the following is correct 
\begin{equation}
	\Pr( |y_i| > (1+\epsilon )c\gamma ) = \frac{1}{2} - \Omega(\epsilon) 
\end{equation}

Then, using Chernoff bound (refer to Lecture 7), we know that a $(\epsilon, \delta)$ estimate requires $m=O(\frac{1}{\epsilon^2}\log \frac{1}{\delta})$. 

Here is an intuitive explanation of the result for $m$. For the probability density distribution $f$ of $|y_i|$, the density at $x = 0$ is constant, and so does the density at the median point $x_{med}$ where $\int_{0}^{x_{med}} f(x)dx = 1/2$. Now, if you have $m$ balls and draw them randomly according to distribution $f$, then, the total  number of balls that lie less than $x_{med}$ are $\frac{m}{2} + O(\sqrt{m})$. Now, for example, suppose we have $\frac{m}{2}-\sqrt{m}$ that fall below $x_{med}$, then, in order to find the median ball, we need to move slightly right from $x_{med}$ to cover another $\sqrt{m}$ balls. Notice that in total we have $m$ balls, that is saying, the increased mass due to moving $x_{med}$ to the right should be approximately $1/\sqrt{m}$. Furthermore, since the density at $x_{med}$ is constant, the distance moving right is $O(1/\sqrt{m})$. Therefore, in order to achieve an $\epsilon$ approximation, $m = O(\frac{1}{\epsilon^2})$ is required.

\subsection{Other $p$ in the interval $(1,2)$}
For other $p$, we can use the similar methodology as we did when $p=1$, i.e., we build a $p$-stable distribution $\mathcal{D}$. If all the elements of $A_{i,:}$ are sampled from $\mathcal{D}(1)$, then, one can show that $y_i$ follows distribution $\mathcal{D}(\|x\|_p)$, and all the rest parts of the algorithm remains the same.

\section{Lower Bound for $p>2$}
Proving the lower bound for $p>2$ will be shown in next class. We will use ideas from information theory (Shannon-Hartley theorem) to prove it.

% \bibliographystyle{alpha}

% \begin{thebibliography}{42}
% \iffalse
% \bibitem[AMS99]{AMS99}
% Noga~Alon, Yossi~Matias, Mario~Szegedy.
% \newblock The Space Complexity of Approximating the Frequency Moments.
% \newblock {\em J. Comput. Syst. Sci.}, 58(1):137--147, 1999.
% \fi 
% \end{thebibliography}

\end{document}
