\documentclass[11pt]{article}
\usepackage{amsmath,amssymb,amsthm}
\usepackage{enumitem}
\usepackage{xcolor}

\DeclareMathOperator*{\E}{\mathbb{E}}
\let\Pr\relax
\DeclareMathOperator*{\Pr}{\mathbb{P}}

\newcommand{\eps}{\epsilon}
\newcommand{\inprod}[1]{\left\langle #1 \right\rangle}
\newcommand{\R}{\mathbb{R}}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {\bf CS 388R: Randomized Algorithms } \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
      \textcolor{red}{\textbf{NOTE:} THESE NOTES HAVE NOT BEEN EDITED OR CHECKED FOR CORRECTNESS}
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[4]{\handout{#1}{#2}{#3}{Scribe: #4}{Lecture #1}}

\newtheorem{theorem}{Theorem}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{observation}[theorem]{Observation}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{assumption}[theorem]{Assumption}

% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex

\begin{document}

\lecture{5 --- 09/14/2017}{Fall 2017}{Prof.\ Eric Price}{Satyanarayana, Ajil Jalal}

\section{Overview}
In this lecture we study the Coupon Collector problem and calculate expected and high probability estimates for the problem. We also study the famous Balls in Bins problem and negative dependence among random variables.

\section{Coupon Collector's Problem}
Suppose you go to a coupon shop which has coupons of n different colors. When you buy a coupon, the shopkeeper picks a color (coupon) uniformly at random and sells it. Your goal is to keep on buying coupons until you collect all the n varieties of coupons.
\begin{enumerate}[label=(\alph*)]
    \item Let $T$ be the random variable describing the total number of coupons you buy to collect all the n colors (coupons). What is $E[T]$?
    \item How well does $T$ concentrate about $E[T]$?
\end{enumerate}
\paragraph{Solution (a)}
    Let $Z_i$ be the random variable describing the number of coupons you need to buy to collect a new color after collecting $(i-1)$ colors.

    So, $T = \sum\limits_{i=1}^n Z_i$ and hence $E[T] = \sum\limits_{i=1}^n E[Z_i]$.
    
    After collecting $i-1$ colors, the probability that the shopkeeper picks a new color for the next coupon is $\frac{n-i+1}{n}$. Let this be $p_i$.
    
    We know that, $Pr(Z_i = k) = (1-p_i)^{k-1} p_i$\\
    $E[Z_i] = \sum\limits_{k=1}^{\infty} k (1-p_i)^{k-1} p_i$\\
    Let $S = \sum\limits_{k=1}^{\infty} k (1-p_i)^{k-1}$.
    \begin{equation*}
        \begin{split}
            S & = 1.(1-p_i)^0 + 2.(1-p_i)^1 + 3.(1-p_i)^2 + 4.(1-p_i)^3 + \ldots \\
            (1-p_i)S & = \text{\;\;\;\;\;\;\;} 0 \text{\;\;\;\;\;\;\;\;}     + 1.(1-p_i)^1 + 2.(1-p_i)^2 + 3.(1-p_i)^3 + \ldots \\
            \text{On subtracting them, \;\;\;} p_i.S & = (1 - p_i)^0 + (1 - p_i)^1 + (1 - p_i)^2 + (1 - p_i)^4 +\ldots\\
            p_i.S & = \frac{1}{p_i}\\
            S & = \frac{1}{p_i^2} 
        \end{split}
    \end{equation*}
    Hence, $E[Z_i] = p_i.S = \frac{1}{p_i} = \frac{n}{n-i+1}.$\\
    $E[T]  = \sum\limits_{i=1}^n E[Z_i] = \sum\limits_{i=1}^n \frac{n}{n-i+1} = \sum\limits_{j=1}^n \frac{n}{j} = n H_n \leq n (\ln n + 1)$
    
\paragraph{Solution (b)}
    By Markov's inequality, we know that 
    \begin{equation*}
        \begin{split}
        Pr( T \geq n^2 ) & \leq \frac{E[T]}{n^2} \\
                          & \leq \frac{\ln n + 1}{n}
        \end{split}
    \end{equation*}
    Hence, for large values of $n$, $T$ does not exceed $n^2$ with high probability. Now, let's try to get a better bound with chebysev's inequality $Pr( | T - E[T] | > t ) \leq \frac{Var(T)}{t^2}$, where $Var(T)$ is the variance of $T$.\\
    \begin{equation*}
        \begin{split}
            Var(T) & = Var (\sum\limits_{i=1}^n Z_i) \\
                   & = \sum\limits_{i=1}^n Var(Z_i) \\
                   & = \sum\limits_{i=1}^n \frac{1 - p_i}{p_i^2}\\
                   & = \sum\limits_{i=1}^n n \frac{i-1}{(n-i+1)^2}\\
                   & = n \sum\limits_{i=1}^n \frac{n-i}{i^2}\\
                   & \leq \sum\limits_{i=1}^n \frac{n^2}{i^2}\\
                   & \leq n^2 . \frac{\pi^2}{6} = \Theta(n^2)
        \end{split}
    \end{equation*}
    Hence, $Pr(|T - E[T]| > t) \leq \Theta(\frac{n^2}{t^2})$. So, $T$ lies between $n.H_n - O(n)$ and $n.H_n + O(n)$ with high probability.\\
            
\textit{Note:} To get the tightest bound possible, use the fact that each $Z_i$ is a sub-exponential random variable, and hence $T$ is a sub-gamma variable. This gives $O(n\log n) - O(n\log \frac{1}{\delta})\leq T \leq O(n\log n) + O(n\log \frac{1}{\delta})$ with probability $1-\delta$.

\section{Balls and Bins}
Given $n$ balls thrown uniformly and independently at random into $n$ bins. Let $X_i$ be the random variable denoting the number of balls that land in bin $i$.
\begin{enumerate}[label=(\alph*)]
    \item Find $E[X_i]$.
    \item Find a good Upper bound on $E[\max\limits_i X_i]$.
    \item Find a good Lower bound on $E[\max\limits_i X_i]$.
    \item Find the expected number of empty bins and concetration bounds for it.    
    %\item How well does $\max\limits_i X_i$ concentrate about $E[\max\limits_i X_i]$
\end{enumerate}
\paragraph{Solution (a)}
Let $X_{ij}$ be the indicator random variable denoting whether ball $j$ falls into bin $i$. So, $X_{i} = \sum\limits_{j=1}^n X_{ij}$. 
We know that, $E[X_{ij}] = \frac{1}{n}$. Hence $E[X_{i}] = \sum\limits_{j=1}^n E[X_{ij}] = 1$

\paragraph{Solution (b)}
Let $Y = \max\limits_i X_i$. In order to compute $E[Y]$, we use the following result.
\begin{equation*}
    \begin{split} E[Y] = \sum\limits_k Pr(Y \geq k) \end{split}
\end{equation*}
Hence it suffices to compute $Pr(\max\limits_i X_i \geq k)$ $\forall k$.\\
We know that,
    $$Pr(X_i = k) = {n \choose k} (\frac{1}{n})^k . (1 - \frac{1}{n})^{n-k}$$
By Sterling's approximation which states that $(\frac{n}{k})^k \leq {n \choose k} \leq (\frac{en}{k})^k$,
\begin{equation}
    \begin{split}
        Pr(X_i = k)     & \leq (\frac{en}{k})^k . (\frac{1}{n})^k . 1 \\
                    & = (\frac{e}{k})^k 
    \end{split}
\end{equation}
and since $(1 - \frac{1}{n})^{n-k} \geq \frac{1}{e}$,
\begin{equation}
    \begin{split}
        Pr(X_i = k) & \geq (\frac{n}{k})^k . (\frac{1}{n})^k . (1 - \frac{1}{n})^{n-k} \\
                    & \geq (\frac{n}{k})^k . (\frac{1}{n})^k . (\frac{1}{e}) \\
                    & = \frac{1}{e.k^k}
    \end{split}
\end{equation}

Now let's compute $Pr(\max\limits_i X_i \geq k)$, which is equal to the probability that any of the $X_i$'s exceed $k$. This can be calculated using union bound as follows.
\begin{equation}
    \begin{split}
        Pr(\max\limits_i X_i \geq k) & = Pr(\bigcup\limits_{i=1}^n X_i \geq k) \\
                                & \leq \sum\limits_{i=1}^n Pr(X_i \geq k) \text{\;\;\;\;\;Using union bound}\\
                                & = n.Pr(X_i \geq k) \text{\;\;\;\;\;\;for any\;} i\\
                                & \leq n.\sum\limits_{k' \geq k} Pr(X_i = k')\\
                                & \leq n.\sum\limits_{k' \geq k} \big(\frac{e}{k'}\big)^{k'} \text{\;\;\;\;\;\;From Equation 1}\\
                                & \leq n.\sum\limits_{k' \geq k} \big(\frac{e}{k}\big)^{k'}\\
                                & \leq n. \frac{(\frac{e}{k})^{k}}{1-\frac{e}{k}} \text{\;\;\;\;\;\;\Big(As\;} \sum\limits_{k' \geq k} p^{k'} \leq \sum\limits_{k' = k}^{\infty} p^{k'} = \frac{p^k}{1-p} \text{\Big)}\\
                                & \leq 2n.(\frac{e}{k})^k \text{\;\;for\;} k \geq 6  
    \end{split}
\end{equation}

As stated above, 
\begin{equation*}
    \begin{split}
        E[\max\limits_i X_i] & = \sum_{k=1}^n Pr(\max\limits_i X_i \geq k) \\
                             & \leq \sum_{k=1}^n min(1, 2n\big(\frac{e}{k}\big)^k) \text{\;\;\;\;\;From Inequality 3} 
    \end{split}
\end{equation*}
Let $k^*$ be the minimum value of $k$ such that $2n\Big(\frac{e}{k}\Big) < 1$. Then,
\begin{equation*}
    \begin{split}
        E[\max\limits_i X_i] & \leq \sum_{k=1}^n min(1, 2n\big(\frac{e}{k}\big)^k) \text{\;\;\;\;\;From Inequality 3} \\
                             & \leq (k^*-1) + \sum\limits_{k = k^*}^n 2n\big(\frac{e}{k}\big)^k \\
                             & \leq (k^*-1) + 2\Big(2n\big(\frac{e}{k^*}\big)^{k^*} \Big)\\
                             & \leq k^* -1 + 2\\
                             & \leq k^* + 1
    \end{split}
\end{equation*}
Now let's compute the value of $k^*$.
    $$2n(\frac{e}{k^*})^{k*} < 1 \implies (\frac{e}{k^*})^{k*} < \frac{1}{2n} \implies k^* \log(\frac{k^*}{e}) < \log 2n$$
    It's easy to see that $k^* = \Theta\Big(\frac{\log n}{\log \log n}\Big)$ satisifies the above inequality. Hence, $E[\max\limits_i X_i] \leq \Theta\Big(\frac{\log n}{\log \log n}\Big)$.

\paragraph{Solution (c)}
Using Stirling's approximation, a lower bound on $\Pr(X_i\geq k)$ is,
\begin{equation*}
\begin{split}
\Pr(X_i\geq k) \geq & \left(\frac{n}{k}\right)^k\frac{1}{n^k}\left(1-\frac{1}{n}\right)^{n-k},\\
\geq & \left(\frac{1}{k}\right)^k e^{-1}.
\end{split}
\end{equation*}

For $k=\frac{\log n}{3\log\log n}$, this gives $\Pr(X_i\geq k) \geq \frac{1}{en^{1/3}}$.

The variables $X_i,\, i\in \left[n\right]$ are negatively associated (formally defined in the next section). One implication of negative association among random variables is:
\begin{displaymath}
\Pr(X_i \geq k \mid X_1 < k,\cdots, X_{i-1}<k) \geq \Pr(X_i\geq k)\,\forall\, i\in \left[n\right]
\end{displaymath}

Hence the probability that all $X_i$ are less than $k$ is given by,
\begin{equation*}
\begin{split}
\Pr( \cap_{i=1}^{n} X_i < k) = & \Pr(X_1 < k)\Pr(X_2<k | X_1 < k)\cdots\Pr(X_n<k | X_1<k, X_2<k, \cdots, X_{n-1}<k),\\
	\leq & \left(1-\frac{1}{en^{1/3}}\right)^n=e^{-\Omega(n^{2/3})} \qed.
\end{split}
\end{equation*}

From part (b) and (c), we conclude that with high probability, the maximum number of balls in a bin is within a constant factor of the expected maximum number of balls.

\subsection{Number of Empty Bins}

An extension to the above problem is to ask how many bins are empty in expectation, and how well they concentrate around the expectation.

Let $Z_i=1_{X_i=0}$ indicate the event that bin $i$ is empty, and $Z=\sum_{i=1}^{n}Z_i$ denote the number of empty bins. In this case,
\begin{displaymath}
 \Pr\left[Z_i=1\right]=\left(1-\frac{1}{n}\right)^n \leq \frac{1}{e}\Rightarrow\E\left[Z\right] \leq \frac{n}{e}.
\end{displaymath}

If $X_i$ are independent variables, then $Z_i$ would also be independent Bernoulli variables, and we could apply a Chernoff bound to obtain a high probability estimate of $Z$. However, negative association of $\{Z_i\}_{i=1}^{n}$ allows for application of Chernoff-Hoeffding bounds on $Z$, which takes the form,
\begin{equation*}
	\begin{split}
	\Pr\left[|Z-\E Z |\geq t\right]\leq 2e^{-2t^2/n}.
	\end{split}
\end{equation*}  

Hence $Z\in \left[ \E Z- O\left(\sqrt{n\log\frac{2}{\delta}}\right),\E Z + O\left(\sqrt{n\log\frac{2}{\delta}}\right)\right]$ with probability $1-\delta$.
\section{Negatively Associated Random Variables}
All material for this section can be found in \cite{DD96}.

Let $\mathbf{X}:=\left(X_1,X_2,\cdots,X_n\right)$ be a vector of random variables.

\begin{definition}
	The random variables $\mathbf{X} = \{ X_1, X_2, \ldots, X_n\}$ are negatively associated, if for every two disjoint index sets, $I,J \subseteq\left[n\right]$,
	\begin{displaymath}
		\E\left[f(\mathbf{X}_I)g(\mathbf{X}_J)\right]\leq \E\left[f(\mathbf{X}_I)\right]\E\left[g(\mathbf{X}_J)\right],
	\end{displaymath} 
	for all non-decreasing functions $f$ and $g$.
\end{definition}

Negative association implies the following properties:
\begin{enumerate}
	\item Negative correlation: $\E\left[\prod_{i\in I} x_i\right]\leq \prod_{i\in I}\E\left[ x_i\right]$.
	
	\textit{Proof}: Let $X_I=(X_{i_1},X_{i_2},\cdots,X_{i_k})$ Choose $f(X_{I\setminus{i_k}})=\prod_{i\in I, i\neq i_k}x_i$ and $g(X_{i_k})=x_{i_k}$. This gives $\E\left[\prod_{i\in I} x_i\right]\leq \E\left[\prod_{i\in I, i\neq i_k} x_i\right]\E\left[x_{i_k}\right]$. Apply induction over all $k$ variables.
	
	\item Negative orthants: $\Pr\left[X_i \geq t_i \forall i \in I \right] \leq \prod_{i\in I}\Pr\left[X_i\geq t_i\right]$.
	
	\textit{Proof}: Similar to negative correlation, choose $f,g$ to be indicator functions and apply induction.
	
	\item Chernoff-Hoeffding Bounds: $\E\left[e^{\lambda\sum_i x_i}\right] \leq \prod_{i}\E\left[e^{\lambda x_i}\right]$. (Equality holds if all variables are independent)
	
	\textit{Proof}: Similar to negative correlation, choose $f(X_{I\setminus{i_k}})=\prod_{i\in I, i\neq i_k}e^{\lambda x_i},g(X_{i_k})=e^{\lambda x_{i_k}}$ and apply induction.
	
	\item If $\mathbf{X}$ and $\mathbf{Y}$ are negatively associated individually, and $\mathbf{X},\mathbf{Y}$ are independent, then $(\mathbf{X},\mathbf{Y})$ is jointly negatively associated.
	
	\item For disjoint index sets $I_j\subseteq\left[n\right]$, let $Y_j=f_j(X_{I_j})$, where $f_j$ are all non-decreasing functions. Then $\{Y_j\}$ are negatively associated.
\end{enumerate}

\subsection{Negative Association in Balls and Bins}
\begin{lemma}
	 Zero-One Lemma: If $X_1, X_2,\cdots,X_n \in \{0,1\}$ and $\sum_{i=1}^{n} X_i=1$, then $X_1,X_2,\cdots,X_n$ are negatively associated.
\end{lemma}
\begin{proof}
	Without loss of generality, assume $f(\overrightarrow{0})=0, g(\overrightarrow{0})=0$ (if this is not true, you can add appropriate constants). For disjoint index sets $I,J\subseteq\left[n\right]$, either $f(X_I)=0$ or $g(X_J)=0$, since the index corresponding to the non zero value cannot be in $I$ and $J$ simultaneously. Hence,
	\begin{equation*}
		\begin{split}
		\E\left[f(X_I)g(X_J)\right]=&0,\, \E\left[f(X_I)\right]\geq 0 ,\, \E\left[g(X_J)\right]\geq 0,\\
		\Rightarrow\E\left[f(X_I)g(X_J)\right]\leq&\E\left[f(X_I)\right]\E\left[g(X_J)\right].
		\end{split}
	\end{equation*}
\end{proof}

For the balls and bins problem, let $Y_{ij}:=1$ if ball $i$ lands in bin $j$, and $Y_{ij}=0$ otherwise. By the Zero-One lemma, $\{Y_{ij}\}_{j=1}^{n}$ are negatively associated. Property (4) from the previous subsection implies that $\{Y_{ij}\}_{i=1,j=1}^{i=n,j=n}$ are negatively associated.

Since $X_j=\sum_{i=1}^{n} Y_{ij}$, property (5) from the previous subsection implies that $\{X_j\}_{j=1}^{n}$ are negatively associated. This also implies that the indicator random variables $Z_j=1_{X_j=0}$ are negatively associated.
\bibliographystyle{alpha}

\begin{thebibliography}{42}

% \bibitem[AMS99]{AMS99}
% Noga~Alon, Yossi~Matias, Mario~Szegedy.
% \newblock The Space Complexity of Approximating the Frequency Moments.
% \newblock {\em J. Comput. Syst. Sci.}, 58(1):137--147, 1999.

\bibitem[MR95]{MR95}
\newblock{Rajeev Motwani  and Prabhakar Raghavan}
\newblock {\emph{Randomized Algorithms}}
\newblock {Cambridge University Press, New York}, 1995.

\bibitem[DD96]{DD96}
\newblock{Devdatt Dubhashi  and Desh Ranjan}
\newblock {\emph{Balls and Bins: A Study in Negative Dependence}}
\newblock {BRICS Report Series, University of Aarhus}, 1996.

\end{thebibliography}

\end{document}