\documentclass[11pt]{article}
\usepackage{amsmath,amssymb,amsthm}
\usepackage{xcolor}

\DeclareMathOperator*{\E}{\mathbb{E}}
\let\Pr\relax
\DeclareMathOperator*{\Pr}{\mathbb{P}}

\newcommand{\eps}{\epsilon}
\newcommand{\inprod}[1]{\left\langle #1 \right\rangle}
\newcommand{\R}{\mathbb{R}}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {\bf CS 388R: Randomized Algorithms, Fall 2019 } \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
      \textcolor{red}{\textbf{NOTE:} THESE NOTES HAVE NOT BEEN EDITED OR CHECKED FOR CORRECTNESS}
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[5]{\handout{XX}{#3}{#4}{Scribe: #5}{Lecture #1: #2}}

\newtheorem{theorem}{Theorem}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{observation}[theorem]{Observation}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{assumption}[theorem]{Assumption}

% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex

\begin{document}

\lecture{9}{Limited Independence}{September 26th, 2019}{Prof.\ Eric Price}{Yangxinyu Xie, Sabee Grewal}
% Example:
%\lecture{1}{Coupon Collector; Balls and Bins}{August 29th, 2019}{Prof.\ Eric Price}{John Doe, Jane Roe}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%Overview%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Overview}

We begin by listing the upsides and downsides of fully independent families of hash functions and compare them with the case of limited independence. 

\paragraph{Full Independence}
\begin{enumerate}
    \item[+] Let's you analyze things really well because you know exactly what's happening. You can do tight analysis (with means, variances, concentrations (cf. Chernoff bounds)).
    \item[+] Assuming full independence is fairly accurate even if your hash function is not. For example, non-adversarial input (for example, high entropy inputs) and cryptographic functions are examples where assuming full independence is justified.
    \item[$-$] You cannot easily store/lookup your hash values. Of course, depending on the situation you may not care. Take load balancing, for example. You're given a task that needs to be assigned to some machine. You can randomly your machine, assign the task, and your job is complete. Whereas with a hash table, you must be able to lookup  your item once it's stored. 
\end{enumerate}

\paragraph{Limited Independence}
\begin{enumerate}
    \item[+] You can get guarantees for all inputs, even if they're adversarial. 
    \item[+] You can store the hash function. 
    \item[+] You can typically still analyze means, variances, and $O(1)$-moments. 
    \item[$-$] You cannot compute $O(\log n)$-moments (which means you cannot use Chernoff bounds.) 
\end{enumerate}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%Introduction and Definitions%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Introduction and Definitions}

\begin{definition}
    A family of hash functions $\mathcal{H}$ is called universal if $h_i: \mathcal{U} \to [m]$ such that $\forall x,y \in \mathcal{U}, x\ne y, a, b \in[m]$
    \begin{equation}
        \mathbb{P}[h(x) = h(y)] \le \frac{1}{m}
    \end{equation}
\end{definition}

\begin{definition}
    A family of hash functions $\mathcal{H}$ is called approximately universal if $h_i: \mathcal{U} \to [m]$ such that $\forall x,y \in \mathcal{U}, x\ne y, a, b \in[m]$ and some $\epsilon > 0,$
    \begin{equation}
        \mathbb{P}[h(x) = h(y)] \le \frac{1+\epsilon}{m}
    \end{equation}
\end{definition}

\begin{definition}
    A family of hash functions $\mathcal{H}$ is called pairwise independent if $h_i: \mathcal{U} \to [m]$ such that $\forall x,y \in \mathcal{U}, x\ne y, a, b \in[m]$
    \begin{equation}
        \mathbb{P}[h(x) = a, h(y) = b] \le \frac{1}{m^2}
    \end{equation}
\end{definition}

\begin{definition}
    A family of hash functions $\mathcal{H}$ is called approximately pairwise independent if $h_i: \mathcal{U} \to [m]$ such that $\forall x,y \in \mathcal{U}, x\ne y, a, b \in[m]$ and some $\epsilon > 0,$
    \begin{equation}
        \mathbb{P}[h(x) = a, h(y) = b] \le \frac{1+\epsilon}{m^2}
    \end{equation}
\end{definition}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%Standard Hash Table%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Standard Hash Table}

The key claim to fame for standard hash tables is expected constant time query while only using linear space. In this section, we'll discuss space requirements of hash tables and show that pairwise independence (or, in fact, universality) suffices for constant time queries. 

Let $\mathcal{U}$ be a universe of items and $S$ be the subset of words that we're trying to store. We're going to map these items under a function $h \in \mathcal{H}$ to a table of size $m$. That is, for each item in $S$, our hash function maps it to some index from $1$ to $m$. We'll handle collisions by chaining items together with linked lists. 

\textit{Space complexity.} A single word takes $\log\lvert\mathcal{U}\rvert$ bits to store and we are trying to store $\lvert S \rvert$ words. Additionally, we need $\log \lvert \mathcal{H}\rvert$ bits to store our hash function. So, in total we need $O(\rvert S \lvert \log \lvert \mathcal{U} \rvert + \log \lvert \mathcal{H} \rvert)$ space. If we can make our hash function sufficiently small, then our space complexity is linear in the number of words we're trying to store (i.e. we require $O(\lvert S \rvert)$ space). In the next section we'll see that we can get a small hash family (and therefore be able to construct a standard hash table with linear space) when we have pairwise independence.

\textit{Time complexity.}
How long does it take to lookup an $x \in \mathcal{U}$? 
Roughly speaking, the total query time scales with the length of the linked list at a given cell (i.e. the number of elements $y \in S \mid h(x) = h(y)$) plus time it takes to compute $h(x)$. More formally, let $T$ be the time it takes to lookup an $x \in \mathcal{U}$ and let $T_{h(x)}$ be the time to compute $h(x)$. We're interested in the expectation of $T$ over some random hash function from some family $\mathcal{H}$. We can solve for $\underset{h \in \mathcal{H}}{\E}[T]$ like so: 

\begin{equation*}
    \begin{split}
        \underset{h \in \mathcal{H}}{\E[T]} &\leq T_{h(x)} + \sum_{y \in S} \underset{h \in \mathcal{H}}{\Pr}[h(x) = h(y)] \\
        \underset{h \in \mathcal{H}}{\E[T]} &\leq T_{h(x)} + \sum_{y \in S} \frac{2}{m} \\
        \underset{h \in \mathcal{H}}{\E[T]} &\leq T_{h(x)} + 1 + \frac{2}{m}\lvert S \rvert\\
        \underset{h \in \mathcal{H}}{\E[T]} &\leq T_{h(x)} + O(1) \\
        \underset{h \in \mathcal{H}}{\E[T]} &\leq O(1).
    \end{split}
\end{equation*}

The first step follows from the fact that $h$ is from a universal family of hash functions (see Definitions 1 and 2). In our case, we use Definition 2 with $\epsilon = 1$. The last step follows from the assumption that computing our hash function takes constant time.  

Our analysis shows that queries to our table complete in constant time in expectation. 
However, we rely on the fact that our hash functions are pairwise independent and can be stored efficiently. In the next section, we discuss how this is accomplished.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%Pairwise Independent Hash Functions%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Pairwise Independent Hash Functions}

Let $M = 2^m$ and $U = 2^u$, we fill a matrix $A \in \mathbb{R}^{m\times u}$ with random bits and then pick a random vector $b \in \{0,1\}^m$. Define a hash family $\mathcal{H}$ in field $\mathbb{F}_2$
\begin{equation}
    \mathcal{H} = \{h_{A,b}(x) = Ax+b\}
\end{equation}

The space of used by the hash function is the space to store $A$ and $b$, which is $O(um) + O(u) = O(um)$. The time it takes to compute is again $O(um)$.
\begin{theorem}
    \label{tmpi}
    $\mathcal{H}$ is pairwise independent.
\end{theorem}
\begin{proof}
    Given $x,y\in \{0,1\}^{u}$, suppose $h_{A,b}(x) = \alpha, h_{A,b}(y) = \beta, x \ne y$ where $\alpha, \beta \in \{0,1\}^{m}$. Let $z = x - y$ and $\gamma = \alpha - \beta$, we have $z \ne \{0\}^m$ since $x \ne y$ and $Az = \gamma$. Therefore, we have
    \begin{align*}
    \mathbb{P}[A_1 z = \gamma_1] & = 1/2 \\
    \mathbb{P}[A_2 z = \gamma_2|A_1] & = 1/2\\
    \vdots\\
    \mathbb{P}[A_m z = \gamma_m|A_{m - 1}, \ldots, A_1] & = 1/2
    \end{align*}
    Thus,
    \begin{equation}
        \mathbb{P}[A z = \gamma]  = \mathbb{P}[A_m z = \gamma_m|A_{m - 1}, \ldots, A_1]\ldots\mathbb{P}[A_2 z = \gamma_2|A_1]\mathbb{P}[A_1 z = \gamma_1] = (1/2)^m
    \end{equation}
    Given $A$, we have $b = \alpha - Ax$. Since $b$ and $A$ are chosen independently, we have $\mathbb{P}[b = \alpha - Ax] = (1/2)^m$. Overall, \begin{equation}
        \mathbb{P}[h_{A,b}(x) = \alpha, h_{A,b}(y) = \beta] = (1/2)^m (1/2)^m = (1/2)^{2m}
    \end{equation}
\end{proof}


\begin{corollary}
    $\mathcal{H}$ is universal.
\end{corollary}
\begin{proof}
    By theorem \ref{tmpi}, we have 
    \begin{equation}
        \mathbb{P}[h_{A,b}(x) = \alpha, h_{A,b}(y) = \beta] = (1/2)^{2m}
    \end{equation}
    Let $\alpha = \beta$, we then have $2^m$ choices of $\alpha \in \{0,1\}^m$, then
    \begin{equation}
        \mathbb{P}[h_{A,b}(x) = h_{A,b}(y)] = (1/2)^{2m}2^m = (1/2)^{m}
    \end{equation}
\end{proof}

\subsection{Random Toeplitz Matrix}
An improvement of the above construction is to use Toeplitz Matrix. We construct a matrix $A \in \mathbb{R}^{m\times u}$ by populating the first row and first column of $A$ with random $\{0,1\}$ entries and letting any other entry $A_{ij} = A_{i-1, j-1}$. Theorem \label{tmpi} still holds true in this construction. To see this, we have
    \begin{align*}
    A_{2,1}z_1=\gamma_2 - (A_{2,2}z_2 + ... + A_{2,m}z_m)
    &= \gamma_2 - (A_{1,1}z_2 + ... + A_{2, m-1}z_m)\\
    \vdots\\
    \end{align*}
Since the first column is picked at random, we have again
    \begin{align*}
    \mathbb{P}[A_2 z = \gamma_2|A_1] & = 1/2\\
    \vdots\\
    \mathbb{P}[A_m z = \gamma_m|A_{m - 1}, \ldots, A_1] & = 1/2
    \end{align*}
The space of used by the hash function is the space to store the first row and the first column of $A$ as well as $b$, which is $O(u+m)$. The time it takes to compute is till $O(um)$.

\subsection{A 2-approximate Universal Hashing}
Pick a random odd number $a$ in $[M]$. We define a hash family $\mathcal{H}$
\begin{equation}
    \mathcal{H} = \{h_{a}(x) = (ax \mod U) / 2^{u - m}\}
\end{equation}
The following theorem can be shown, we direct the reader to \cite{DHKP97} for the proof.
\begin{theorem}
    \begin{equation}
        \mathbb{P}[h(x)=h(y)] \le 2/M
    \end{equation}
\end{theorem}

\section{k-wise Independent Hash Functions}

Again, let $M = 2^m$ and $U = 2^u$, consider the hash family
\begin{equation}
    \mathcal{H} = \{h_{a}(x) = (ax + b) \mod M\}
\end{equation}
where $a,b$ are chosen independently.

\subsection{$M = U$}

It is easy to see that this hash family is pairwise independent. For $x,y \in [U], x \ne y$
\begin{equation}
    \mathbb{P}[ax + b = \alpha,  a'x+b' = \beta] = (\frac{1}{M})^2 = (\frac{1}{U})^2
\end{equation}

The time to calculate the hash function is $O(1)$ and the space to store the hash function is also $O(1)$.

\subsubsection{$M < U$}

As $M$ is less than $U$, we have the hash family is $(1 + M/U)$ approximately pairwise. 
\begin{equation}
    \mathbb{P}[ax + b = \alpha,  a'x+b' = \beta] = (\frac{|S|}{U})^2 =  (\frac{1}{M})^2(1 \pm \frac{M}{U})^2
\end{equation}
where $\lfloor \frac{U}{M}\rfloor \le S \le \lceil \frac{U}{M}\rceil$.

\subsection{k-wise independent hash function}

Consider instead a generalised hash family

\begin{equation}
    \mathcal{H} = \{h_{a}(x) = \sum_{i = 0}^{k - 1}a_ix^i \mod M\}
\end{equation}
where $a_i$ is independently generated from $ [M] $ for all $i$.

The proof of k-wise independence is similar to above.

\bibliographystyle{alpha}

\begin{thebibliography}{42}

\bibitem[CW79]{CW79}
Lawrence~J~Carter and Mark~N~Wegman.
\newblock Universal classes of hash functions.
\newblock {\em Journal of computer and system sciences}, 18(2):143--154, 1979.

\bibitem[DHKP97]{DHKP97}
Martin~Dietzfelbinger, Torben~Hagerup, Jyrki~Katajainen, and Martti~Penttonen.
\newblock A reliable randomized algorithm for the closest-pair problem.
\newblock {\em J. Algorithms}, 25(1):19–51, 1997.

\end{thebibliography}

\end{document}