\documentclass[11pt]{article}
\usepackage{amsmath,amssymb,amsthm}
\usepackage{xcolor}
\usepackage{mathtools}

\DeclareMathOperator*{\E}{\mathbb{E}}
\let\Pr\relax
\DeclareMathOperator*{\Pr}{\mathbb{P}}
\DeclareMathOperator*{\Var}{\mathrm{Var}}

\newcommand{\eps}{\epsilon}
\newcommand{\inprod}[1]{\left\langle #1 \right\rangle}
\newcommand{\R}{\mathbb{R}}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {\bf CS 388R: Randomized Algorithms, Fall 2019 } \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
      \textcolor{red}{\textbf{NOTE:} THESE NOTES HAVE NOT BEEN EDITED OR CHEKED FOR CORECTNES}
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[5]{\handout{XX}{#3}{#4}{Scribe: #5}{Lecture #1: #2}}

\newtheorem{theorem}{Theorem}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{observation}[theorem]{Observation}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{assumption}[theorem]{Assumption}

% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex

\begin{document}

\lecture{5}{Coupon Collector; Balls and Bins}{September 12}{Prof.\ Eric Price}{James Dong, Jack Youstra}
% Example:
%\lecture{1}{Coupon Collector; Balls and Bins}{August 29th, 2019}{Prof.\ Eric Price}{John Doe, Jane Roe}

\section{Coupon Collector}

Problem description: Let's say a certain cereal company is selling cereal boxes with one of $n$
different figures. How many cereal boxes do you need to buy in order to collect all the figures?
Let $n$ be the number of figures. Question: how long does this take?

\subsection{Expected number of draws}

Idea: let $Z_i$ be the time until the next new item when $i$ items are unseen.
Then $T = Z_n + \dots + Z_1$, and by linearity of expectation we have
$\E[T] = \sum_{i=1}^n \E[Z_i]$.
Each $Z_i$ is the number of random draws with probability $i/n$, so
$Z_i \sim \operatorname{Geometric}(i/n)$, and $\E[Z_i] = n/i$.
\[ \E[T] = n \sum_{i=1}^n 1/i = n H_n \le n (\log n + 1). \]

\subsection{Concentration bounds}

Now we want to find concentration bounds. Is it likely that this process will take a lot of draws?

First try: Markov's inequality gives $T \le 3 n H_n$ with probability $2/3$. For
error probability $1/n$, we need $n^2 H_n$ draws.

Next try: Chebyshev's inequality: $\Pr[|T - n H_n| \ge t] \le \sigma^2/t^2$.
$\sigma^2 = \Var(T) = \sum_{i=1}^n \Var(Z_i)$ since the time we found one item does not influence
how many more draws you need until the next item, so $Z_i$ are independent.
From Wikipedia, we have $\Var(Z_i) = n (n - i)/i^2$, so
\[ \sigma^2 = n \sum \frac{n - i}{i^2} \le n^2 \sum 1/i^2 \le n^2 \pi^2/6. \]
So $\sigma = O(n)$, so $\Pr[|T - n H_n| \ge t n] \le \pi^2/6 t^2$.

Note: to get the tightest bound, make use of the fact that each $Z_i$ is geometric and thus subexponential,
so $T = \sum Z_i$ is subgamma (i.e.\ $\E[e^{\lambda x}] \le e^{\lambda^2 \sigma^2/2}$ for all $\lambda \le B$ for some bound $B$), which somehow implies that the tail is exponential.

\subsection{Alternative concentration bound}

\[ \Pr[\text{element $i$ not seen by time $T$}] = (1 - 1/n)^T, \]
so by union bound
\[ \Pr[\text{any element not seen by time $T$}] \le n (1 - 1/n)^T \approx n e^{-T/n}. \]

\section{Balls and Bins}

We throw $n$ balls into $n$ bins.
\[ X_i \coloneqq \text{\# balls in bin $i$} \]
\textbf{Questions:} $\E[X_i]$? $\E[\max X_i]$? $\E[\text{empty bins}]$? Concentration?

\subsection{Expectation of each $X_i$}

We know $\sum X_i = n$, so by linearity of expectation $\E[X_i] = 1$.

\subsection{Concentration of $\max X_i$}

Turns out it's easier to look at concentration first than to derive expectation.

Let's look at $\Pr[X_i = k] = \binom{n}{k} \frac{1}{n^k} \left( 1 - \frac1n \right)^{n-k}$.

\textbf{Key property:}
\[
    \left(\frac{n}{k}\right)^k \le \binom{n}{k} \le \left(\frac{en}{k}\right)^k.
\]
The left side follows from the fact that
\[
    \binom{n}{k} = \frac{n}{k}\cdot \frac{n-1}{k-1} \dots \frac{n - k + 1}{1},
\]
and each individual fraction is less than $n/k$.
The right side follows from Stirling's approximation.

Then $\Pr[X_i = k] \le (en/k)^k (1/n) (1-1/n)^{n-k} \le (e/k)^k.$

$\Pr[X_i \ge k] = \sum_{j=k}^{\infty} (e/j)^j \le 2(e/k)^k$ if $k \ge 6$.

Then by union bound \[ \Pr[\max X_i \ge k] \le n \Pr[X_i \ge k] \le 2n (e/k)^k. \]
For this to be less than a constant, we have $\Pr[\max X_i \ge k] \le 1/2$ whenever $(k/e)^k \ge O(n)$.
Turns out if $k = \Theta(\log n / \log \log n)$, we have $(k/e)^k \ge \sqrt{\log n}^k = (\log n)^{\frac12 k}
= (\log n)^{\Theta(\log_{\log n} n)} = n.$

\section{Negative association}

\begin{definition}
A set of random variables $x_{1}, ..., x_{n}$ is negatively associated (N.A.) if for all disjoint subsets $I,J\subseteq [n]$, and for all monotonically nondecreasing (a mirror argument holds for monotonically nonincreasing) $f$, $g$, the following inequality holds

$$\E[f(X_{I}) \cdot g(X_{J})] \leq \E[f(X_{I})]\E[g(X_{J})]$$

This means it concentrates at least as well as independent variables, and one variable tends to be smaller when another is bigger.
\end{definition}

\subsection{Zero-one lemma}

\begin{lemma}
If $x \in \{0, 1\}$ and $\sum x_{i} = 1$, then $x$ is negatively associated.
\end{lemma}


\begin{proof}
Without loss of generality we assume that $f(0)=g(0)=0$. In fact for any constant $c$ we have
\begin{align*}
  \E[\big(f(X_I)+c\big)\cdot g(X_J)]&=\E[f(X_I)\cdot g(X_I)]+c\E[g(X_J)]\\
  \E[f(X_I)+c]\E[g(X_J)]&=\E[f(X_I)]\E[g(X_J)]+c\E[g(X_J)].
\end{align*}
Hence a translation of $f$ does not affect the correctness of inequality. This arguments also works for function $g$. Thus we can always assume $f(0)=g(0)=0$.

For all inputs, $f(x_{i})$, $g(x_{j}) \geq 0$.
$$\E[f(X_{I})g(X_{J})] = 0 \leq \E[f(X_{I})]\E[g(X_{J})].$$
The first equality comes from the fact that either $X_{I} = 0$ or $X_{J} = 0$.
\end{proof}

\subsection{Composition rules}

\begin{enumerate}

\item If have N.A. random variables and apply monotonically nondecreasing function, the application of the function creates a new N.A. set of random variables.

\item If $X$, $Y$ are individually N.A. and independent, then $(X, Y)$ is N.A.

\end{enumerate}

This relates back to balls in bins!

Take $W_{i, j} = 1$ iff ball $i$ lands in bin $j$. Then,

\begin{enumerate}
\item All $W_{i, *}$ are negatively associated with each other, and
\item $W_{*, j}$ is also negatively associated.
\end{enumerate}
Even though $Z_{i}$ isn't independent, we can still use the Chernoff bound because the Chernoff bound is based on the moment-generating function which changes little for negative associativity.
$$\text{Independence: } \E(e^{\lambda(\sum z_{i}  - \mu_{i})}) = \prod_{i} \E(e^{\lambda(z_{i}  - \mu_{i})})$$
$$\text{Negative associativity: } \E(e^{\lambda(\sum z_{i}  - \mu_{i})}) \leq \prod_{i} \E(e^{\lambda(z_{i}  - \mu_{i})})$$



\end{document}