\documentclass[11pt]{article}
\usepackage{latexsym}
\usepackage{amsmath,amssymb,amsthm}
\usepackage{tikz}
\usetikzlibrary{arrows}

\DeclareMathOperator*{\E}{\mathbb{E}}
\let\Pr\relax
\DeclareMathOperator*{\Pr}{\mathbb{P}}
\newcommand{\R}{\mathbb{R}}

\newcommand{\eps}{\epsilon}
\newcommand{\inprod}[1]{\left\langle #1 \right\rangle}
\newcommand{\poly}{{\rm poly}}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {\bf CS 395T: Sublinear Algorithms } \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[4]{\handout{#1}{#2}{#3}{Scribe: #4}{Lecture #1 }}

\newtheorem{thm}{Theorem}[section]
\newtheorem{claim}[thm]{Claim}
\newtheorem{lemma}[thm]{Lemma}
\newtheorem{define}[thm]{Definition}
\newtheorem{cor}[thm]{Corollary}
\newtheorem{obs}[thm]{Observation}
\newtheorem{remark}[thm]{Remark}
\newtheorem{example}[thm]{Example}
\newtheorem{construction}[thm]{Construction}
\newtheorem{conjecture}{Conjecture}
\newtheorem{THM}{Theorem}
\newtheorem{COR}[THM]{Corollary}
\newtheorem{question}{Question}
\newtheorem{fact}[thm]{Fact}
\newtheorem{prop}[thm]{Proposition}
% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex

\begin{document}

\lecture{7 --- Sept. 15, 2016}{Fall 2016}{Prof.\ Eric Price}{Zhao Liu, Changyong Hu }

\section{Overview}

In today's lecture, we will discuss the following problems:

\begin{enumerate}
\item Count-Min Review \cite{CM05}
\item Count-Sketch \cite{CCF02}
\item Fourier analysis of Count-Sketch \cite{MP14}
\end{enumerate}


\section{Count-Min Review}
 
{\bf Sketch}:

Consider the problem for finding heavy hitters of a histogram $x$ of a data stream, i.e. the $k$ most frequent items ($x_k$ is large). Assume the items are turnstile in $[n]$. Last time, we use $R$ different hashes $y^{(r)}$ $(r = 1,...,R)$ with each $y^{(r)}$ served as a counter of size $B = O(k)$. This is equivalent to think about storing a ``table'' with $R$ rows and $B$ columns. Each row of the table stands for a counter $y^{(r)}$, and
a counter $y_i^{(r)}$ for each cell $(r, i)$ of the table. The following is a summary:

\begin{enumerate}
\item Choose $R$ pair-wise independent hash functions $h_1, h_2,
  \cdots, h_R : [n] \rightarrow [B]$.
\item For each hash function $h_r$, the counters $y_i^{(r)} = \underset{u,h_r(u) = i}{\sum} x_u$, where $ r \in [R], i \in [B].$
\end{enumerate}

This is a linear function of $x$, so it can be expressed as a matrix. Given $y$, because it's an overestimate, in order to let failure probability decay exponentially, our recovered estimate $\hat{x}$ of $x$ is taken as the minimum of each hash.

{\bf Recovery Algorithm}
\begin{enumerate}
\item In each row, estimate $\hat{x}_u^{(r)} = y_{h_r(u)}^{(r)}$.
\item Overall, estimate $\hat{x}_u = \min_r \hat{x}_u^{(r)}$.
\end{enumerate} 
The error and space for count-min is
\begin{equation*}
\| \hat{x} - x \|_\infty \leq \frac{\|x - x_k \|_1}{k} ~with ~probability ~1 - \frac{1}{n}, space of ~O(k\log(n)).
\end{equation*}

The intuition of this algorithm is trying to separate large terms from small terms and making use of sparsity. We briefly review the analysis of error here:

{\bf Analysis}:

Let $H=\{1,\cdots,k\}$ be the indices of k most frequent elements and $T=\{k+1,\cdots,n\}$ be the rest. For a particular hash function $h_r$ and element $i$:

\begin{equation*}
\begin{split}
|\hat{x}_i^{(r)} - x_i | & = \underset{j\in H, h_r(i)=h_r(j)}{\sum} x_j + \underset{j\in T, h_r(i)=h_r(j)}{\sum} x_j \\
& \leq \underbrace{0}_{with~ probability ~1 - \frac{k}{B}} +\underbrace{\|x_T \|_1 /B}_{in ~expectation}\\
& \leq \underbrace{0}_{with~ probability ~ \frac{9}{10}} +\underbrace{ \frac{\|x_T \|_1}{k} }_{with~probability~\frac{9}{10}}\\
\end{split}
\end{equation*}
if we set $B=10k$. Thus for each $r$ and $i$, by a union bound we have
\begin{equation*}
\hat{x}_i^{(r)} - x_i=| \hat{x}_i^{(r)} - x_i | \leq \frac{\| x_T\|_1}{k} ~with~ probability ~\frac{8}{10}.
\end{equation*}

This implies that

\begin{equation*}
\hat{x}_i = \underset{r}{\min}~ \hat{x}_i^{(r)} \leq x_i + \frac{\|x_T\|_1}{k}~ with ~probability ~1-(\frac{1}{5})^R
\end{equation*}

Choose $R= O(\log n)$, then
\begin{equation*}
BR = O(k\log n), ~ 1-(\frac{1}{5})^R = 1-n^{-c}, ~where ~c ~is ~a ~constant ~value.
\end{equation*}

What if some coordinates are negative? For some error $\sigma = O(\|x_T\|_1/k)$, we still have 
$Pr[|\hat{x}_i^{(r)} - x_i |\leq \sigma] \geq \frac{4}{5}$.  Then
after $R$ samples, with $1-e^{-O(R)}$ probability we will have that at
least $\frac{n}{2}$ of the $\hat{x}_i^{(r)}$ will land in $x_i \pm
\sigma$. Their median then lands in that region. So "count-median" will work in this case.

\section{Count-Sketch}


\subsection{Problem}
Suppose $x$ has one large element $(O(n))$ and $n-1$ small elements $(O(1))$. With count-min algorithm:
\begin{itemize}
\item $\hat{x_u}\approx \frac{n}{B} $ if $x_u$ is not the big element.
\item $\hat{x_u}\approx C+\frac{n}{B}$ if $x_u$ is the big element.
\end{itemize}

Then the error will approximately be $\| \hat{x} - x \|_\infty \approx \frac{n}{B}$. We want to get a more precise estimation of error like, for example,  $\sqrt{\frac{n}{B}}$.


\subsection{Setup}

The idea of count-sketch is to introduce random signs in the summation, so that
the errors tend to cancel each other out. For example, think about the case when the $O(1)$ elements are $\pm 1$ in the count-min algorithm. Then $\hat{x}_u$ has mean 0 (or $c$) and standard deviation of $O(\sqrt{\frac{n}{B}})$ (because $\hat{x}_u$ is sum of about $\frac{n}{B}$ $x_i$'s.)

{\bf Sketch}:

\begin{enumerate}
\item Choose $R$ pair-wise independent hash functions $h_1, h_2,
  \cdots, h_R : [n] \rightarrow [B]$ and $s_1, \dotsc, s_R : [n] \to \{\pm 1\}$. For each hash function $h_r$, we need $B$ counters.
\item $\forall r \in [R], \forall i \in [B], y_i^{(r)} = \underset{u,s.t.h_r(u) = i}{\sum} s_r(u) x_u$.
\end{enumerate}

It is a linear function of $x$, which can be expressed as a matrix (with random entries in $\{\pm 1\}$).
Given $y$, we recover our estimate $\hat{x}$ of $x$ by:

{\bf Recovery Algorithm}

\begin{enumerate}
\item In each row, estimate $\hat{x}_u^{(r)} = s_r(u)y_{h_r(u)}^{(r)}$. 
\item Overall, estimate $\hat{x}_u = \underset{r\in [R]}{\text{median}} ~\hat{x}_u^{(r)}$.
\end{enumerate}

The only difference from Count-Min is the introduction of random signs
$s_r$, and the use of the median for estimation. As follows, we estimate its error.

\subsection{Analysis of error}

First, let's bound the term $| \hat{x}_u^{(r)} - x_u |$ for any $r\in [R]$ by upper bounding its variance:
\begin{equation*}
\begin{split}
\mathbb{E}[(\hat{x}_u^{(r)} - x_u)^2] & = \mathbb{E}[(\underset{u' \neq u, h(u') = h(u)}{\sum}x_{u'}\cdot s_r(u') \cdot s_r(u))^2] \\
& = \mathbb{E}[\underset{u' \neq u, h(u') = h(u)}{\sum}{x_{u'}}^2 \cdot 1_{h(u')=h(u)} + \underset{u' \neq u'' \neq u, h(u')=h(u'')=h(u)}{\sum}{x_{u'}x_{u''}s_r(u')s_r(u'')}] \\
& = \frac{1}{B}\cdot \underset{u' \neq u, h(u')=h(u)}{\sum}{x_{u'}^2} \\
& \leq \frac{\|x\|^2_2}{B}
\end{split}
\end{equation*}

where the third equality follows from pairwise independence of $s_r$. Applying Chebyshev Inequality,
\begin{equation*}
Pr\bigl[|\hat{x}_u^{(r)} - x_u| \leq \frac{2\|x\|_2}{\sqrt{B}}\bigr] \geq \frac{3}{4}
\end{equation*}

But the above upper bound is not optimal. It is reasonable to think that after applying median, as $R\rightarrow\infty$, the error will converge to 0. 

We claim that usually, we have
\begin{equation*}
|\hat{x}_u^{(r)} - x_u| \ll \frac{\|x\|_2}{\sqrt{B}}
\end{equation*}


To show this claim and estimate the error, we start with considering a simplified example: sampling from independent normal distribution. Let $z_1,z_2,\cdots,z_R \sim N(0,1)$, $z^* = \underset{r\in [R]}{\text{median}} ~z_r$. How will the median $z^*$ decay with $R$? Let 
$V_r:=1_{\{\text{event}~ z_r\geq\varepsilon\}}$. Because $z_r$'s are symmetric,
\begin{equation*}
\begin{split}
Pr[|z^*| > \varepsilon] & \leq 2Pr[z^* > \varepsilon]  \leq 2Pr[at ~least~ \frac{R}{2} ~of~ z_r ~are~ \geq \varepsilon]
\end{split}
\end{equation*}
\begin{equation*}
\begin{split}
\mathbb{E}[V_r] =& Pr[z_r\geq \epsilon]\\
=&\frac{1}{2}\cdot Pr[0\leq z_r \leq \varepsilon] \\
=& \frac{1}{2} - \Omega(\varepsilon)
\end{split}
\end{equation*}
where the last equality follows from $Pr[0\leq z_r\leq \epsilon] = \int_{0}^{\epsilon}\frac{1}{\sqrt{2\pi}}e^{-\frac{t^2}{2}}dt\approx O(\frac{\epsilon}{\sqrt{2\pi}})$. Therefore,
\begin{equation*}
E[{\sum}V_r] \leq R(\frac{1}{2} - \Theta(\varepsilon))
\end{equation*}
From Chernoff bound,
\begin{equation*}
Pr\bigl[{\sum}V_r - E[{\sum}V_r] \geq \Theta(\varepsilon R)\bigr] \leq e^{-\frac{\varepsilon^2R^2}{2R}}
\end{equation*}
Thus we have 
\begin{equation*}
Pr[|z^*|\geq\varepsilon] 
\leq Pr\bigl[{\sum}V_r\geq \frac{R}{2}\bigr]
\leq 2e^{-\Theta(\varepsilon^2R)}
\end{equation*}
We can set $\varepsilon = \frac{1}{\sqrt{R}}$ for constant probability.

If the random variables $z_r$ are not Gaussian, with some appropriate conditions (symmetric and concentration of probability around 0), following similar arguments as above, we have 
\begin{thm}\
Let $z_i$ be independent variables symmetric around 0, with $E[{z_i}^2]=1$ and $Pr[|z_i|\leq\varepsilon] \gtrsim \varepsilon$ for any $\epsilon$ less than some constant, then their median
\begin{equation*}
|z^*|\lesssim\frac{1}{\sqrt{R}} ~with~ \frac{3}{4} ~probability. 
\end{equation*}

\end{thm}

Back to our case, let $z^{(r)} = \hat{x}_u^{(r)} - x_u =
\underset{u'= u, h(u') = h(u)}{\sum}x_{u'}\cdot s_r(u') \cdot s_r(u)$. These are symmetric random variables. To satisfy the conditions of above theorem, one can resort to Fourier Analysis described below.


\section{Fourier Analysis of Count-Sketch\cite{MP14}}
You can actually give a tighter analysis of Count-Sketch, which shows
that \emph{most} coordinates are estimated to higher precision, if
your hash functions are fully independent. As we described in an
earlier class, the assumption of fully independent hash functions is
unfortunate, but it can be justified using cryptographic hash
functions and computational assumptions on the adversarial input, or
assuming the input has high entropy.


Note that the details of this analysis also can be found in Eric
Price's presentation slide of SODA'2015. Here is the link :
http://www.cs.utexas.edu/~ecprice/slides/concentration-slides.pdf.

\begin{thm}
Let $z$ be symmetric random variable and $E[z^2]=1$, if $\mathcal{F}_z(t) \geq 0, \forall t$, then 
\begin{equation*}
Pr[|z|\leq \varepsilon] \gtrsim \varepsilon
\end{equation*}
\end{thm}

\begin{thm}
  Assume that $h$ and $s$ are fully independent hash functions, and
  consider the output $\hat{x}$ of Count-Sketch.  Then $\forall t \leq
  R$, we have
\begin{equation*}
|\hat{x}_i - x_i| \leq \sqrt{\frac{t}{R}} \cdot \frac{\| x_T\|_2}{\sqrt{k}}
\end{equation*}
with probability $1-e^{-\Omega(t)}$.

This implies that $\E[\hat{x}_i-x_i]\leq \frac{1}{\sqrt{R}}\cdot
\frac{\|x_T\|_2}{\sqrt{k}}$ after excluding $e^{-\Omega(R)}$ events.
\end{thm}

\begin{thm}
For any set S of size K,
\begin{equation*}
Pr\bigl[\|\hat{x}_s - x_s\|_2 \geq O(c)\cdot \sqrt{\frac{K}{R}}\cdot \sigma\bigr] \leq \frac{1}{K^c} ~for~ any~ c > 0.
\end{equation*}
\end{thm}



\bibliographystyle{alpha}

\begin{thebibliography}{42}

\bibitem[CCF02]{CCF02}
Moses~Charikar, Kevin~Chen, and Martin~Farach-Colton.
\newblock Finding Frequent Items in Data Streams.
\newblock {\em ICALP}, 2002.

\bibitem[CM05]{CM05}
Graham~Cormode and S.~Muthukrishnan.
\newblock Summarizing and Mining Skewed Data Streams.
\newblock {\em SDM}, 2005.

\bibitem[MP14]{MP14}
Gregory~T.~Minton and Eric~Price.
\newblock Improved Concentration Bounds for Count-Sketch
\newblock {\em SODA(best student paper)} 2014.



\end{thebibliography}


\end{document}






%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

