\documentclass[11pt]{article}
\usepackage{amsmath,amssymb,amsthm,url}

\DeclareMathOperator*{\E}{\mathbb{E}}
\let\Pr\relax
\DeclareMathOperator*{\Pr}{\mathbb{P}}

\newcommand{\eps}{\epsilon}
\newcommand{\inprod}[1]{\left\langle #1 \right\rangle}
\newcommand{\R}{\mathbb{R}}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {\bf CS 388R: Randomized Algorithms } \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[4]{\handout{#1}{#2}{#3}{Scribe: #4}{Lecture #1}}

\newtheorem{theorem}{Theorem}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{observation}[theorem]{Observation}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{assumption}[theorem]{Assumption}

% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex

\urldef{\hoeffding}\url{https://en.wikipedia.org/wiki/Hoeffding%27s_inequality}

\begin{document}

\lecture{4 --- Sep 9, 2015}{Fall 2015}{Prof.\ Eric Price}{Xiangru Huang $\&$
Chad Voegele}

\section{Overview}

In previous lectures, we introduced some basic probability, the Chernoff bound,
the coupon collector problem, and game tree evaluation.

In this lecture, we will introduce concentration inequalities.

\section{Coupon Collector Problem}
Draw numbers (coupons) independently from $[n] = \{1, 2, \ldots, n\}$.
How long does it take to see all of the numbers?

Suppose $T_i$ is the number of draws to get the $i$-th new number. Let $T =
\sum_i T_i$.

\begin{fact}
 The $T_i$'s are independent of each other.
\end{fact}

\begin{fact}
 $T_i$ follows geometric distribution with success probability, $p = \frac{n+1-i}{n}$.
\end{fact}

\begin{fact}
 If $X \sim Geometric(p)$, 
  \begin{align*}
    E[X] &= p \cdot 1 + (1-p) \cdot (E[X | X \ge 2]) \\ 
         &= p + (1-p) (1+E[X]) \\
 \Rightarrow E[X] &= \frac{1}{p}
  \end{align*}
 \begin{align*}
   E[X^2] &= p \cdot 1^2 + (1-p) \cdot E[X^2 | X \ge 2] \\
          &= p + (1-p) E[(X+1)^2] \\
          &= p + (1-p) (E[X^2] + 2E[X] + 1) \\
          &= p + (1-p) E[X^2] + 2(1-p)/p + (1-p) \\
 \Rightarrow E[X^2] &= \frac{2-p}{p^2} 
 \end{align*}
  \begin{align*}
    Var(X) &= E[(X-E(X))^2] = E[X^2] - (E[X])^2\\
  \Rightarrow Var(X) &= \frac{1-p}{p^2} \le \frac{1}{p^2}
  \end{align*}
\end{fact}

Therefore in the Coupon Collector Problem,
  \begin{align*}
    E[T] &= \sum_{i=1}^{n} E[T_i] = \sum_{i=1}^{n} \frac{n}{n+1-i} = n H_n
    \approx n \log n
  \end{align*}
  \begin{align*}
    Var[T_i] &\le \frac{1}{p_i^2} = \left(\frac{n}{n+1-i}\right)^2\\
    \Rightarrow Var[T] &= \sum_{i=1}^{n} Var[T_i]
    \le n^2 \left(\sum_{i=1}^{n} \frac{1}{i^2}\right) 
    \le n^2 \cdot \frac{\pi^2}{6} = O(n^2)
  \end{align*}

\section{Concentration Inequalities}

$$
\forall i,~Pr[T_i \ge 1 + \alpha] \le \left(1 - \frac{n+1-i}{n}\right) ^ \alpha
$$
Assume $\delta$ is some failure probability.
Setting $\alpha_i = \left(\frac{n}{n+1-i}\right) \log \frac{n}{\delta}$ and
because $(1-\frac{1}{x})^x < \frac{1}{e}$, we have
$$
\forall i,~Pr[T_i \ge 1 + \alpha_i] \le \frac{\delta}{n}
$$
\begin{definition}
 Union Bound
 $$
 Pr[X_1 \cup X_2 \cup \ldots \cup X_n] \le \sum_i Pr[X_i]
 $$
\end{definition}
Using a union bound, we have
\begin{align*}
  Pr&\left[T \ge n + n \log n \log \frac{n}{\delta}\right]\\
  &= Pr\left[\sum_i T_i \ge n + \sum_i \alpha_i\right]\\
  &= Pr[T_1 \ge 1 + \alpha_1 \cup \ldots \cup T_n \ge 1 + \alpha_n]\\
  & \le \sum_i Pr[T_i \ge 1 + \alpha_i] \\
  & \le \delta
\end{align*}

\begin{definition}
 With High Probability (w.h.p.)
 $$
 X \le O(y) \mbox{ w.h.p. } \Leftrightarrow \forall c_2, ~ \exists c_1,
 \mbox{ s.t. } Pr[X \le c_1 y] \le n^{-c_2} 
 $$
\end{definition}

$T = O(n \log^2 n)$ with high probability.

\subsection{Markov's Inequality}
For a non-negative random variable $T$ and any non-negative $\alpha$,
\begin{align*}
  & E[T] \ge Pr[T \ge \alpha] \cdot \alpha\\
  &\Rightarrow Pr[T \ge \alpha] \le \frac{E[T]}{\alpha}
\end{align*}

In the Coupon Collector Problem, 
\begin{align*}
  &\alpha = \frac{E[T]}{\delta} = \frac{n H_n}{\delta}\\
  &\Rightarrow Pr\left[T \ge \frac{n H_n}{\delta}\right] \le \delta
\end{align*}

\subsection{Chebyshev's Inequality}
For a random variable, X, let $\mu = E[X]$ denote the expectation and $\sigma^2
= Var[X]$ denote the variance. Starting from Markov's Inequality, we find
$$
  Pr[(X - \mu)^2 \ge \alpha^2] \le \frac{E[(X-\mu)]^2}{\alpha^2} = \frac{\sigma^2}{\alpha^2}
$$
Setting $\alpha \rightarrow \alpha \sigma$
$$
Pr[(X - \mu)^2 \ge \alpha^2 \sigma^2] \le \frac{1}{\alpha^2}
$$
Taking the square root, we find
$$
Pr[X \ge \mu + \alpha \sigma] \le \frac{1}{\alpha^2}
$$
$$
Pr[X \le \mu - \alpha \sigma] \le \frac{1}{\alpha^2}
$$
Using this result in the Coupon Collector Problem, gives us
$$
Pr[T \ge n H_n + \frac{1}{\sqrt{\delta}} O(n)] \le \delta
$$
Setting $\delta = \frac{1}{log^2n}$
$$
Pr[T \ge n H_n + O(n \log n)] \le O\left(\frac{1}{log^2 n}\right)
$$
Most of the time, the typical deviation is $O(\sigma)$.
$$Pr[|x-\mu| \le O(\sigma)] \approx 1-\delta$$

\subsection{Moment Method}
If $f$ is non-negative, by Markov's inequality,
$$
Pr[f(X-\mu) \ge f(\alpha)] \le \frac{E[f(X-\mu)]}{f(\alpha)}
$$
For $f$ increasing,
$$
Pr[X-\mu \ge \alpha] \le \frac{E[f(X-\mu)]}{f(\alpha)}
$$
Set $f = |t|^k$, 
$$
Pr[|X-\mu|^k \ge |\alpha|^k] \le \frac{E[|x-\mu|^k]}{|\alpha|^k}
$$
For one side,
$$
Pr[X \ge \mu + \alpha] \le \frac{E[|x-\mu|^k]}{|\alpha|^k}
$$

Setting $\delta = \frac{E[|x-\mu|^k]}{|\alpha|^k}$, we have
$$Pr\left[X \le \mu + E[|x-\mu|^k]^{1/k} \cdot 
\left(\frac{1}{\delta}\right)^{1/k}\right] \ge 1 - \delta$$

If we consider $X \sim N(0, \sigma^2)$, we know
$$
E[|x|^k] \approx (k \sigma^2)^{k/2} ~ \forall k > 0
$$
which means
$$
  Pr\left[X \ge \mu + 
    O\left(\sqrt{k} \cdot \sigma \cdot
  \left(\frac{1}{\delta}\right)^{1/k}\right)\right] 
    \le \delta
$$
Setting $k = \log\frac{1}{\delta}$, we get
$$
  Pr\left[X \ge \mu +
  O\left(\sqrt{\log\frac{1}{\delta}}\right)\right] \le \delta
$$

\subsection{Moment Generating Function}
\begin{definition}
  The moment generating function, parameterized by $\lambda$, is defined as
  $$
  MGF_X(\lambda) = E[e^{\lambda(X-\mu)}]
  $$
\end{definition}

Assume $X$ is centered ($E[X] = 0$).
$$
e^{\lambda x} = 1 + \lambda x + \frac{(\lambda x)^2}{2} + \frac{(\lambda x)^3}{3!} + \ldots + \frac{(\lambda x)^k}{k!}
$$
We can use parameter $\lambda$ to adjust the weights on each term. When
$\lambda$ is larger, more weight is on higher order terms.

From the derivation of the Moment Method, setting $f(x) = e^{\lambda x}$,
$$
Pr[X \ge \mu + \alpha] \le \frac{MGF_X(\lambda)}{e^{\lambda \alpha}}
$$

\begin{fact}
 If $X \sim N(0, \sigma^2)$, $MGF_X(\lambda) = E[e^{\lambda x}] \le e^{\lambda^2
 \sigma^2} ~ \forall ~ \lambda \in \R$
\end{fact}
Using this,
\begin{align*}
  Pr[X \ge \mu + \alpha] &\le \frac{MGF_x(\lambda)}{e^{\lambda \alpha}} \\
                         &=e^{\frac{\lambda^2 \sigma^2}{2} - \lambda \alpha}\\
                         &= e^{\frac{1}{2}
  \left(\lambda \sigma - \frac{\alpha}{\sigma}\right)^2 
  - \frac{\alpha^2}{2\sigma^2} }
\end{align*}
Set $\lambda = \frac{\alpha}{\sigma^2}$, we get
$$
Pr[X \ge \mu + \alpha] \le e^{-\frac{\alpha^2}{2\sigma^2}}
$$
If $\delta = e^{-\frac{\alpha^2}{2\sigma^2}}$, we have
$$
\alpha = \sigma\sqrt{2 log \frac{1}{\delta}}
$$
Note that this is the same $O\left(\sqrt{\log\frac{1}{\delta}}\right)$ bound
as we found in the method of moments, except that now we know the constant.

\subsection{Subgaussian Variables}
\begin{claim}
 The following three statements are equivalent if we only care up to a constant
 for $\sigma$ (i.e. $\forall i,j \in \{1,2,3\}, ~\sigma_i = \theta(\sigma_j)$)
 \begin{align}
   &X \mbox{ is subgaussian with parameter } \sigma,~i.e.~
  \forall \lambda \in \R, ~MGF_X(\lambda) \le e^{\frac{\lambda^2
\sigma_1^2}{2}}\label{eq:first}\\
&Pr[X \ge \mu + t] \le e^{-\frac{t^2}{2\sigma_2^2}}\label{eq:second}\\
&E[|x|^k]^{1/k} \le O\left(\sigma_3
    \sqrt{k}\right)\label{eq:third}
 \end{align}
\end{claim}

\begin{fact}
 The sum of subgaussian variables are subgaussian. 
 \begin{align*}
   X &= X_1 + \ldots + X_n\\
  MGF_X(\lambda) &= E\left[e^{\lambda X}\right] = E\left[e^{\lambda(\sum_i
  x_i)}\right] = E\left[\prod_i e^{\lambda x_i}\right] \\
  &= \prod_i E\left[e^{\lambda x_i}\right] \mbox{ (by independence)} \\
		 &= \prod_i MGF_{X_i}(\lambda) \\
		 &\le \prod_i e^{\lambda ^2 \sigma_i^2/2} \mbox{ (by subgaussian)} \\
		 &= e^{\frac{\lambda^2}{2}(\sum_i \sigma_i^2)}
 \end{align*}
 This implies $X$ is subgaussian with parameter $\sqrt{\sum_i \sigma_i^2}$.
\end{fact}

\begin{fact}
  If $X \in [0, 1]$, then $X$ is subgaussian with $\sigma = 1/2$ by Hoeffding's
  Lemma.\footnote{\hoeffding}

 Let
$X = \sum_i X_i $ where $~X_i \in [0,1]$.
$X$ is subgaussian with $\sigma = \sqrt{n}/2$. Plug this into \eqref{eq:second}, we have
$$
Pr[x \ge \mu + \alpha] \le e^{-\frac{2 \alpha^2}{n}}
$$
which is exactly the Chernoff bound.
\end{fact}

\section{Next Class}
In the Coupon Collector Problem, we had
$$
Pr[T_n \ge \alpha] \le \left(1 - \frac{1}{n}\right)^\alpha \approx e^{-\alpha}
$$
This is not of the form $e^{-\alpha^2}$ so we cannot use the subgaussian
results. We will relax the subgaussian requirement to subexponential and
subgamma. This will lead to Bernstein's inequality.

$$
MGF_X(\lambda) = E[e^{\lambda X}] \le e^{\frac{\lambda^2 \sigma^2}{2}} ~
\forall~ |\lambda| \le B
$$

\bibliographystyle{alpha}
\begin{thebibliography}{42}

% \bibitem[AMS99]{AMS99}
% Noga~Alon, Yossi~Matias, Mario~Szegedy.
% \newblock The Space Complexity of Approximating the Frequency Moments.
% \newblock {\em J. Comput. Syst. Sci.}, 58(1):137--147, 1999.

 \bibitem[MR]{MR}
 Rajeev~Motwani, Prabhakar~Raghavan
 \newblock Randomized Algorithms.
 \newblock {\em Cambridge University Press}, 0-521-47465-5, 1995. %pub - isbn - year

%@book{Motwani:1995:RA:211390,
% author = {Motwani, Rajeev and Raghavan, Prabhakar},
% title = {Randomized Algorithms},
% year = {1995},
% isbn = {0-521-47465-5, 9780521474658},
% publisher = {Cambridge University Press},
% address = {New York, NY, USA},
%}

 \bibitem[RV]{RV}
 Roman~Vershynin
 \newblock Introduction to the non-asymptotic analysis of random matrices.
 \newblock {\em CoRR}, abs-1011-3027, 2010.

%@article{DBLP:journals/corr/abs-1011-3027,
%  author    = {Roman Vershynin},
%  title     = {Introduction to the non-asymptotic analysis of random matrices},
%  journal   = {CoRR},
%  volume    = {abs/1011.3027},
%  year      = {2010},
%  url       = {http://arxiv.org/abs/1011.3027},
%  timestamp = {Mon, 05 Dec 2011 18:05:29 +0100},
%  biburl    = {http://dblp.uni-trier.de/rec/bib/journals/corr/abs-1011-3027},
%  bibsource = {dblp computer science bibliography, http://dblp.org}
%}

%notes on BPP, amplification, Ppoly, adlemans theorem:
%http://people.csail.mit.edu/madhu/ST07/scribe/lect11.pdf
\end{thebibliography}

\end{document}
