\documentclass[11pt]{article}
\usepackage{amsmath,amssymb,amsthm}
\usepackage{xcolor}
\usepackage{algorithm}
\usepackage{algpseudocode}
\usepackage{hyperref}
\usepackage{cleveref}
\usepackage{bbm}
\algtext*{EndProcedure}
\algtext*{EndFor}
\algtext*{EndIf}

\DeclareMathOperator*{\E}{\mathbb{E}}
\let\Pr\relax
\DeclareMathOperator*{\Pr}{\mathbb{P}}

\newcommand{\eps}{\epsilon}
\newcommand{\inprod}[1]{\left\langle #1 \right\rangle}
\newcommand{\R}{\mathbb{R}}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 6.45in { {\bf CS 388R: Randomized Algorithms, Fall 2019 } \hfill #2 }
      \vspace{4mm}
      \hbox to 6.45in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 6.45in { {\em #3 \hfill #4} }
      \textcolor{red}{\textbf{NOTE:} THESE NOTES HAVE NOT BEEN EDITED OR CHECKED FOR CORRECTNESS}
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[5]{\handout{XX}{#3}{#4}{Scribe: #5}{Lecture #1: #2}}

\newtheorem{theorem}{Theorem}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{observation}[theorem]{Observation}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{assumption}[theorem]{Assumption}

% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex

\begin{document}

\lecture{19}{Streaming (continued)}{Oct. 31, 2019}{Prof.\ Eric Price}{Haoran Zhang, Sung Yeon Choi}

\section{Introduction}
Given a universe $\mathbb{U} = [n]$ and incoming stream $\{x_1 ... x_m\}$, we defined the frequency of each element by $f_j = |\{x_i| x_i = j\}|$. Last class, we learned how to estimate $||f||_0$, which shows a number of distinct elements in the stream. $||f||_1$ is number of elements in the stream (and thus can be calculated easily using O(log m) space). In this lecture, we will see how to estimate $||f||_2$.

\section{Estimating 2-norm - AMS algorithm}

Suppose we are interested in finding the second moment of frequencies. The easiest solution would be to keep count of all possible elements that come in during the stream then calculate the moment. This will take $n\log{m}$ space. Suppose that we don't have that much space and would like to use only $O(\log{n})$ space to estimate $||f||_2$, or more formally, find the value of an estimate $\hat{F_2}$ such that,
\begin{equation}
    (1-\epsilon)||f||_2 \leq \hat{F_2} \leq (1+\epsilon)||f||_2 \qquad \text{w.p. $1-\delta$}
\end{equation}
We will achieve this by using AMS algorithm, which uses the idea of Johnson-Lindenstrauss lemma we saw in Lecture 17.
\subsection{Johnson-Lindenstrauss}
Recall that Johnson-Lindenstrauss states $\exists{A \in \mathbb{R}^{k x n}}$ where $A_{ij} \sim N(0, \frac{1}{k})$ such that, for sequence $\{x_1 ... x_n\}$, 
\begin{equation}
    (1-\epsilon)||x||_2 \leq ||Ax||_2 \leq (1+\epsilon)||x||_2 \qquad \text{w.p. $1-\delta$}
\end{equation}
where $k = O(\frac{1}{\epsilon^2}\log{\frac{2}{\delta}})$.

It seems that we can use directly to estimate $||f||_2$, but there is a problem: matrix A can be large. We use the intuition from JL lemma to achieve the next algorithm with a good bound.
\subsection{AMS algorithm}
AMS algorithm let us estimate $||f||_2$ by maintaining a linear sketch of $f$. Idea of algorithm is,
\begin{itemize}
    \item Pick $\{s_1 ... s_n\} \sim \{-1, 1\}$  using 4-wise independent hash functions
    \item Calculate $Y = \sum_{i\in[n]} s_if_i$
    \item Return $Y^2$
\end{itemize}
Notice that since frequency is updated linearly, $Y = \sum_{i\in[n]} s_if_i$ is equivalent to $Y = \sum_{j\in[m]} h(x_j)$ where h is 4-wise independent hash function, and therefore does not require any space to compute other than the space required for hash family. 

Let us now look at the expectation of Y to see if the algorithm actually outputs the values we want.
\begin{align*}
    \mathbb{E}[Y] & = \sum_{i\in[n]} \mathbb{E}[s_i]f_i  \\
        & = 0 \\
    \mathbb{E}[Y^2] & = \mathbb{E}[\sum_{i\in[n]}s_i^2f_i^2 + \sum_{i\neq j}s_is_jf_if_j] \\
            & = ||f||_2^2 \\
    \mathbb{E}[Y^4] & \leq 2||f||_2^4 \\
    Var(Y^2) & = \mathbb{E}[Y^4] - \mathbb{E}[Y^2]^2 \\
        & \leq \mathbb{E}[Y^2]^2
\end{align*}

From above, we see that the expected value of $Y^2$ is $||f||_2^2$, which gives us the 2-norm estimation of frequency. Applying Chebyshev's inequality and median of means technique, we get $O(\frac{1}{\epsilon^2}\log\frac{1}{\delta})$ space used like in JL lemma. Since constructing the hash family takes $O(\log n)$ space, we need $O(\frac{1}{\epsilon^2}\log\frac{1}{\delta}\log n)$ space in total, which is what we aimed for.


\section{Heavy Hitter}
Suppose we are interested in finding the set of elements in the stream that appear frequently (i.e. the heavy hitters). More formally, we can define the problem that given a stream $x_1, \dots, x_m$, let frequency $f_j = |\{x_i| x_i = j\}|$, we want to find the set $H_\alpha$ of elements  that appears more than $\alpha m$ times,
\begin{equation}
    H_\alpha = \{i \in [n]| f_i \geq \alpha m\}
\end{equation}
where $\alpha \in (0, 1)$.
\subsection{Insertion-only}
We start with the insertion only model. For simplicity, let $\alpha = \frac{1}{k}$, the goal is to find $H_{\frac{1}{k}}$. With \textbf{Misra-Gries} algorithm, we can solve this problem deterministically. More specifically, we keep at most $k-1$ counters of $(j, \hat{f}_j)$. As scanning the stream, we modify the counters according to each incoming $x_i$
\begin{algorithm}
\caption{Misra-Gries Algorithm}
\label{alg:m-g}
\begin{algorithmic}
\Procedure{Misra-Gries}{k}
\State $M = \emptyset$
\For {$i$ in $[m]$}
    \If {$x_i = j \in M$}
        \State $\hat{f}_j += 1$
    \Else
        \If {$|M| < k - 1$}
            \State Insert new counter $(j, 1)$
        \Else
            \State $\forall j \in M$, $(j, \hat{f}_j) \leftarrow (j, \hat{f}_j - 1)$
            \If {$\hat{f}_j == 0$}
                \State Remove $(j, \hat{f}_j)$
            \EndIf
        \EndIf
    \EndIf
\EndFor
\Return{M}
\EndProcedure
\end{algorithmic}
\end{algorithm}

\begin{claim}\label{c1}
$H_{\frac{1}{k}} \subseteq M$
\end{claim}
\begin{proof}
In another word, with Misra-Gries algorithm, we have no false negatives, but it is possible to have false positive.

As shown in \cref{alg:m-g}, the counters are only decremented when $M$ is full. For each time, $k$ items are decremented together ($k-1$ items in $M$ and the incoming item). With $m$ counts in total, we can only have at most $\frac{m}{k}$ decrements. Thus, if $f_j > \frac{m}{k}$, $\hat{f}_j > 0$. 
\end{proof}
 
\begin{claim}
$f_i - \frac{m}{k} \leq \hat{f_i} \leq f_i$
\end{claim}
\begin{proof}
Similar to the proof of \cref{c1}, with at most $\frac{m}{k}$ decrements, it can be easily shown that such inequality holds.
\end{proof}

\subsection{Strict turnstile - Count Min Sketch}
However, if deletion is allowed, Misra-Gries algorithm can be easily fooled. How should we solve the problem in this case?

Consider a hash fucntion $h: [n] \rightarrow [B]$. Then $Y_{h(j)} = \sum_{j, h(j) = u} f_j$ is the number of items in bucket $u \in B$. Let $\hat{f}_j = Y_{h(j)}$ be the estimator of $f_j$, we can have
\begin{equation}
\begin{aligned}
    \mathbb{E}[|f_j - \hat{f}_j|] &= \mathbb{E}[\sum_{k \neq j} f_k \cdot \mathbbm{1}_{h(k) = h(j)}]\\
    &\leq \frac{||f||_1}{B}
\end{aligned}
\end{equation}
Let $B = \frac{\alpha}{4}$, by Markov's inequality we can have
\begin{equation}
    Pr(|f_j - \hat{f}_j| \geq \alpha||f||_1) \leq \frac{1}{4}
\end{equation}
We can augment such result by running with $r = O(\log n)$ pairwise independent hash functions and let $\hat{f}_j = \min_r Y_{h(j), r}$, which gives us $f_j \leq \hat{f}_j \leq f_j + \alpha||f||_1$ w.h.p.
\end{document}