\documentclass[11pt]{article}
\usepackage{amsmath,amssymb,amsthm}

\DeclareMathOperator*{\E}{\mathbb{E}}
\let\Pr\relax
\DeclareMathOperator*{\Pr}{\mathbb{P}}

\newcommand{\eps}{\epsilon}
\newcommand{\inprod}[1]{\left\langle #1 \right\rangle}
\newcommand{\R}{\mathbb{R}}
\newcommand{\poly}{{\rm poly}}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {\bf CS 395T: Sublinear Algorithms } \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[4]{\handout{#1}{#2}{#3}{Scribe: #4}{Lecture #1 }}

\newtheorem{thm}{Theorem}[section]
\newtheorem{claim}[thm]{Claim}
\newtheorem{lemma}[thm]{Lemma}
\newtheorem{define}[thm]{Definition}
\newtheorem{cor}[thm]{Corollary}
\newtheorem{obs}[thm]{Observation}
\newtheorem{remark}[thm]{Remark}
\newtheorem{example}[thm]{Example}
\newtheorem{construction}[thm]{Construction}
\newtheorem{conjecture}{Conjecture}
\newtheorem{THM}{Theorem}
\newtheorem{COR}[THM]{Corollary}
\newtheorem{question}{Question}
\newtheorem{fact}[thm]{Fact}
\newtheorem{prop}[thm]{Proposition}
% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex

\begin{document}

\lecture{3 --- Sept. 4, 2014}{Fall 2014}{Prof.\ Eric Price}{Zhao Song }

In today's lecture, we will discuss the following problems:

\begin{enumerate}
\item Distinct elements
\item Turnstile model
\item AMS - sketch
\end{enumerate}

\section{Distinct elements}
\begin{description}
\item[Given] $1, 5, 4, 4, 19, \cdots, ~ \in [n]$.
\item[Goal] Estimate $k$(= \# distinct elements) up to factor $(1\pm\epsilon)$ with $1-\delta$ probability.
\end{description}

In order to solve the above problem, let's look at the following basic question:

\begin{description}
\item[Given] t.
\item[Goal] Ask either $k\leq t$ or $k \geq 2t$?
\end{description}

Choose a subset $S \subseteq [n]$, then $\forall i \in [n], i\in s$
with probability $\frac{1}{t}$.  Record whether the intersection of
``stream'' and set $S$ is empty, let $x$ denote this event
$(\text{stream} \cap S)\neq \emptyset$. (Note that $S$ is chosen
before you see a stream of integers).

\begin{equation*}
Pr[x ~is ~ true] = Pr[x] = 1-(1-\frac{1}{t})^k
\end{equation*}
(Note that, $Pr[x]$ is a monotonically increasing function on $k$ when $t$ is fixed)

For $k \leq t$, we have
\begin{equation*}
Pr[x | k\leq t] \leq 1 - (1-\frac{1}{t})^t \approx 1- \frac{1}{e} \approx 0.63
\end{equation*}

For $k \geq 2t$, we have
\begin{equation*}
Pr[x | k \geq 2t] \geq 1 - (1-\frac{1}{t})^{2t} \approx 1 - \frac{1}{e^2} \approx 0.8
\end{equation*}

Repeat to get $x_1, x_2, \cdots, x_m$ independent samples.  Return
whether $\sum_i x_i \geq 0.7 m$.  Since $x_i \in \{0,1\}$ is
subgaussian for $\sigma=\frac{1}{2}$, we have that $\sum_i x_i$ is
also a subgaussian with $\sigma = \frac{\sqrt{m}}{2}$.
\begin{equation*}
\begin{split}
& Pr[\sum_i x_i \geq \mu +t ] \leq e^{\frac{t^2}{2\sigma^2}} = e^{\frac{2t^2}{m}} \\
& Pr[\sum_i x_i \geq 0.63m + 0.07m] \leq e^{2\cdot 0.07^2 m}
\end{split}
\end{equation*}

Therefore with $m = \Theta(\log (1/\delta))$, we can distinguish the
to cases with $1-\delta$ probability.

{\bf Question:} But: how do we store a concise description of $S$?
There can be $2^n$ such sets, and roughly $\binom{n}{k}$ ``likely''
sets.  So storing $S$ would dominate the space complexity.

\begin{description}
\item[Answer 1:] Crypto $h ~: ~\text{SHA-256}$ \cite{SHA256}, or any
  other crypto hash, and choose roughly $S= \{i | \frac{h(i)}{2^{256}}
  <\frac{1}{t}\}$.  Then there would exist streams that break the
  algorithm, but it's (hopefully) computationally intractible to find
  them.

\item[Answer 2:] $h : $ pair-wise independent, $s= \{ i | h(i) < \frac{1}{t} \}$
\end{description}

Let's look at the general definition of some hash functions first,

\begin{define}
Family $H$ of functions from $[n] \rightarrow [m]$ is pair-wise independent if with probability
\begin{equation*}
\underset{h \in H ~\\ x,y \in [n]~\\c,d \in[m]}{Pr}[ h(x) = c~ and~ h(y) = d] =\frac{1}{m^2}
\end{equation*}
\end{define}

\begin{example}
Canonical example: $h(x) = ax + b \pmod m$, where $(a,b) \in [m]$ pair-wise independent if $m$ is a prime $\geq n$.
\end{example}

Let's consider an algorithm that uses pair-wise independent hash function:

{\bf Algorithm:} Let $H$ denote a pairwise-independent hash function
family, choose $h\in H$ such that $h : [n] \rightarrow [B]$, where
$B=\Theta(t)$(the constant will be decided later). Consider the set $S=
\{i | h(i) =0\}$.

Then, for the probability of any $x\in S$, we have an upper bound by the union bound:
\begin{equation*}
Pr[any ~ x \in s] \leq \sum_{i} Pr[i\in s] = \frac{k}{B}
\end{equation*}

And we have a lower bound by Inclusion-Exclusion\footnote{{http://en.wikipedia.org/wiki/Inclusion-exclusion\_principle}}:
\begin{equation*}
\begin{split}
Pr[any ~ x \in s] & \geq \sum_{i} Pr[i\in s] - \sum_{i,j} Pr[i\in s ~and ~j \in s] \\
&= \frac{k}{B} - \frac{k(k-1)}{2B^2} \\
&= \frac{k}{B}(1-\frac{k-1}{B})
\end{split}
\end{equation*}

Let's set $B = 4t$, for $k\leq t$, we have 
\begin{equation*}
Pr[any ~ x\in S] \leq \frac{t}{B} = \frac{1}{4}
\end{equation*}
For $k\geq 2t$, we have
\begin{equation*}
Pr[any ~ x\in S]\geq \frac{1}{2} (1-\frac{1}{4}) = \frac{3}{8}
\end{equation*}

For any $t$, do $\log (\frac{1}{\delta})$ independent
samples/examples, each uses $O(\log n)$ spaces. Since there are
$O(\log n)$ different $t$s, then $O(\frac{1}{\epsilon^2}\cdot \log
(\frac{\log n}{\delta}))$ total space is used to perform distinct
elements.

\newpage

{{\bf Idealized streaming algorithm}\footnote{The details of ISA can be found in Lecture 2 of Course Algorithm for Big Data at Harvard. http://people.seas.harvard.edu/~minilek/cs229r/lec/lec2.pdf}}




We now explain the LogLog algorithm of \cite{DF03}, which improves the
space complexity from roughly $O(\frac{1}{\epsilon^2} \log n)$ to
$O(\frac{1}{\epsilon^2} \log \log n)$.  One algorithm you could use
for distinct elements is the following:

\begin{enumerate}
\item Pick a random hash function $h:[n]\rightarrow [0,1]$
\item Define $z = \underset{i \in \text{stream}}{\min} h(i)$, then
  $\frac{1}{z}-1 \approx k$.
\end{enumerate} 

The observation is that you don't need to store $z$ exactly; you only
need to remember which of $\log n$ different scales $z$ lies in.


{\bf LogLog algorithm}
\begin{enumerate}
\item Pick a random hash function $h: [n] \rightarrow \{0,1\}$. (Note that $h$ is able to convert a stream of integers to a binary string.)
\item For a string $x\in \{0,1\}^\infty$, define $\rho(x)$ to be the number of leading zeros from left. (In \cite{DF03}, they defined $\rho(x)$ in a similar way, where $\rho(x)$ denotes the position of its first 1-bit, e.g. $\rho(1\cdots)=1$ and $\rho(001\cdots)=3$.)
\item Separate elements into $m$ buckets (Analysis in \cite{DF03} shows that $\epsilon = \frac{1.3}{\sqrt{m}}$, here; $\epsilon= \frac{1.05}{\sqrt{m}}$, for HyperLogLog.)
\item Let $m=2^t$, then the first $t$ binary bits of $x$ denote the index of one of $m$ buckets.
\item Let ${\cal M}$ denote the multiset of hashed values, define $z({\cal M}) = \underset{x\in{\cal M}}{\max} ~\rho(x)$.
\item For each bucket $j$, ignore the first $t$ bits and compute $z_j$.
\item Output $\alpha_m m 2^{\frac{1}{m} \sum z_j}$ to approximate $n$, where $\alpha_m$ is a constant value (defined in \cite{DF03}).
\end{enumerate}

Total Space : $O(\frac{1}{\epsilon^2} \log\log n+ \log n)$, where the first term $\frac{1}{\epsilon^2}\log\log n$ is caused by $m$ buckets and the second term $\log n$ is from hash function.
%We only need to know which of the $\log n$(needs $\log\log n$ bit) buckets it lies in.

%$h(i)$ binary string. Record maximum \# leading zeros over alll $i\in \text{stream}$, then $2^z \approx k$.

%Split stream into $\frac{c}{\epsilon^2}$ substreams, where $c=1.3$ and can be reduced to $1.05$. Record $z_j = \max$ \# leading zeros in substream $j$. Answer $\approx \alpha_m m 2^{\frac{1}{m} \sum z_j}$.
%Thus, total space is $O(\frac{1}{\epsilon^2}\log\log n)$.

There exists a better algorithm : 
\begin{thm}\cite{KNW10}
For a stream of indices in $\{1,2,\cdots,n\}$, the algorithm computes $(1\pm \epsilon)$-approximation using an optimal $O(\frac{1}{\epsilon^2} + \log(n))$ bits of space with $\frac{2}{3}$ success probability, where $0<\epsilon <1$.
\end{thm}
%Kane-Nelson-Wooduff's work\cite{KNS10} used $O(\frac{1}{\epsilon^2} + \log n)$ spaces.

\newpage
\section{Turnstile model}

\begin{enumerate}
\item Pick a vector $x\in \R^n$, start at 0.
\item Read a stream of updates $(\cdots, (i,\alpha_i), \cdots)$, where $i\in [n]$, $\alpha_i$ is the number of elements to be added or deleted.
\item For each $(i,\alpha_i)$, we update $x_i \leftarrow x_i + \alpha_i$.
\item Compute $f(x)$.
\end{enumerate}

%$x \in R^n$, starts at 0, stream of updates $(i,\alpha)$. $x_i \leftarrow x_i + \alpha$. End : compute $f(x)$.

A further restriction is the ``strict'' turnstile model, where $x_i$
is always $\geq 0$, which means the count of any item can not be
negative at any time.

What are examples of $f$ that you might want to compute?  Well,
distinct elements corresponds to
\begin{equation}
f(x) = (\# i | x_i \neq 0) = \|x\|_0 \text{~~~(also called the ``sparsity of x'')}
\end{equation}
One may also ask about other norms, e.g. $\|x\|_1$ and
$\|x\|_2$,  or finding spanning tree, or finding the largest entries.

{\bf Estimate $\|x\|_2$ in turnstile model}

Let $A \in \R^{m \times n}$ be a Johnson-Lindenstrauss matrix, where $A_{ij} \sim \mu(0,\frac{1}{m})$, $m=O(\frac{1}{\epsilon^2}\log(\frac{1}{\delta}))$, $\|Ax\|_2^2 = (1\pm \epsilon) \|x\|_2^2$ with probability $1-\delta$.


Given update $(i,\alpha)$, then we have :
\begin{equation*}
\begin{split}
& \\
& x\leftarrow x + \alpha \cdot e_i \\
&Ax \leftarrow Ax +A\cdot e_i\cdot \alpha\\
&y \leftarrow y + \alpha \cdot (column ~i\in A)\\
\end{split}
\end{equation*}
where $e_i$ is the ``elementary unit vector'', a vector of length $n$
with $ e_i = \underbrace{00\cdots 0}_{i-1} 1 \underbrace{00\cdots
  0}_{n-i}$.  This means we can maintain the linear ``sketch'' $y =
Ax$ under streaming updates to $x$.

This would let us estimate $\|x\|_2$ from a small space sketch $Ax$.
The problem is, to do so requires us to remember $A$, which takes more
than $mn$ bits.  So how do we solve this?  The same way we solved not
being able to store $S$ for distinct elements -- with hashing and
limited independence.

\newpage
\section{AMS - sketch \cite{AMS99}}

\begin{define}
H is a k-wise independent hash family if
\begin{equation*}
\forall i_1\neq i_2 \neq \cdots i_k \in [n]~ and ~\forall j_1, j_2, \cdots, j_k \in [m]
\end{equation*}
\begin{equation*}
\underset{h\in H}{Pr} [h(i_1)=j_1 \land \cdots \land h(i_k)=j_k] = \frac{1}{m^k}
\end{equation*}

\end{define}

{{\bf AMS Algorithm}\footnote{More details also can be found in : Lecture 2 of Course Algorithm for Big Data at Harvard. http://people.seas.harvard.edu/~minilek/cs229r/lec/lec2.pdf ; Lecture 2 of Course Sublinear Algorithms for Big Datasets at the University of Buenos Aires. http://grigory.github.io/files/teaching/sublinear-big-data-2.pdf}}:
\begin{enumerate}
\item Pick a random hash function $h : [n]\rightarrow \{-1,+1\}$ from a four-wise independent family.
\item Let $v_i = h(i)$.
\item Let $y= <v,x>$, output $y^2$.
\item From Lemma 3.1 and 3.2, we know that $y^2$ is an unbiased estimator with variance big-Oh of the square of its expectation.
\item Sample $y^2$ $m_1=O(\frac{1}{\epsilon^2})$ independent times : $\{y_1^2, y_2^2, \cdots, y_{m_1}^2\}$. Use Chebyshev's inequality to obtain a $(1\pm\epsilon)$ approximation with $\frac{2}{3}$ probability.  
\item Let $\overline{y} = \frac{1}{m_1}\overset{m_1}{\underset{i=1}{\sum}} y_i^2$.
\item Sample $\overline{y}$  $m_2 = O(\log(\frac{1}{\delta})) $ independent times : $\{\overline{y}_1,\overline{y}_2,\cdots, \overline{y}_{m_2}\}$. Take the median to get $(1\pm \epsilon)$-approximation with probability $1-\delta$.
\end{enumerate}

{\bf Space Analysis} :
Each of the hash function takes $O(\log n)$ bits to store, and there are $O(\frac{1}{\epsilon^2} \log(\frac{1}{\delta}))$ hash functions in total.


%{\bf Simple Alg. for right exp. bounded var}
%\begin{description}
%\item Repeat $O(\frac{1}{\epsilon^2})$ time
%\begin{description}
%\item mean has small var
%\item good $\sim$ $\frac{3}{4}$ probability
%\item Repeat $O(\log\frac{1}{\delta})$ time. Take median.
%\begin{description}
%\item good $\sim$ $1-\delta$ probability.
%\end{description}
%\end{description}
%\end{description}


\begin{lemma}
$E[y^2] = \| x \|_2^2$
\end{lemma}
\begin{proof}
\begin{equation*}
\begin{split}
E[y^2] & = E[(<v,x>)^2]\\
& = E[\overset{n}{\underset{i=1}{\sum}} v_i^2 x_i^2 + \underset{i\neq j}{\sum} v_iv_jx_ix_j ]\\
& = E[\overset{n}{\underset{i=1}{\sum}} v_i^2 x_i^2] + E[ \underset{i\neq j}{\sum} v_iv_jx_ix_j ] \\
& = \overset{n}{\underset{i=1}{\sum} } x_i^2 + 0 \\
& = \| x\|_2^2 \\
\end{split}
\end{equation*}
where $E[v_iv_j]=E[ v_j]\cdot E[v_k]=0$ since pair-wise independence. 
\end{proof}

\begin{lemma}
$E[ (y^2 - E[y^2] )^2 ] \leq 2 \| x\|_2^4$
\end{lemma}

\begin{proof}
\begin{equation*}
\begin{split}
E[ (y^2 - E[y^2] )^2 ] & = E[(\underset{i\neq j}{\sum} v_iv_jx_ix_j)^2]\\
& = E[4\underset{i< j}{\sum} v_i^2v_j^2x_i^2x_j^2 + 4 \underset{i\neq j\neq k}{\sum} v_i^2v_jv_k x_i^2x_jx_k +24 \underset{i<j<k<l}{\sum} v_iv_jv_kv_l x_ix_j x_k x_l]\\
& = 4 \underset{i< j}{\sum} x_i^2 x_j^2 + 4 \underset{i\neq j\neq k}{\sum} E[v_i^2 v_jv_k x_i^2 x_jx_k] + 24 E[\underset{i<j<k<l}{\sum} v_iv_jv_kv_l x_ix_j x_k x_l ] \\
& = 4 \underset{i< j}{\sum} x_i^2 x_j^2 + 0 +0 \\
& \leq 2 \|x\|_2^4 \\
\end{split}
\end{equation*}
where $E[v_i^2 v_jv_k]=E[ v_j]\cdot[v_k]=0$ since pair-wise independence, 

and $E[v_iv_jv_kv_l] =E[ v_i]E[v_j]E[v_k]E[v_l]=0$ since four-wise independence.
\end{proof}

\bibliographystyle{alpha}

\begin{thebibliography}{42}

\bibitem[AMS99]{AMS99}
Noga~Alon, Yossi~Matias, and Mario~Szegedy.
\newblock The Space Complexity of Approximating the Frequency Moments.
\newblock {\em J. Comput. Syst. Sci.}, 58(1):137--147, 1999.

\bibitem[DF03]{DF03}
Marianne~Durand and Philippe~Flajolet.
\newblock Loglog Counting of Large Cardinalities.
\newblock {\em ESA}, LNCS 2832:605--617, 2003.

\bibitem[KNW10]{KNW10}
Daniel M. Kane, Jelani Nelson, and David P. Woodruff.
\newblock An optimal algorithm for the distinct elements problem.
\newblock {\em Proceedings of the twenty-ninth ACM SIGMOD-SIGACT-SIGART symposium on Principles of database systems.} 2010.

\bibitem[SHA256]{SHA256}
%Noga~Alon, Yossi~Matias, Mario~Szegedy.
\newblock Descriptions of SHA-256, SHA-384, and SHA-512. %The Space Complexity of Approximating the Frequency Moments.
\newblock {\em NIST}, 2014-09-07, http://csrc.nist.gov/groups/STM/cavp/documents/shs/sha256-384-512.pdf





\end{thebibliography}


\end{document}






%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
