\documentclass[11pt]{article}
\usepackage{amsmath,amssymb,amsthm}

\DeclareMathOperator*{\E}{\mathbb{E}}
\let\Pr\relax
\DeclareMathOperator*{\Pr}{\mathbb{P}}

\newcommand{\eps}{\epsilon}
\newcommand{\inprod}[1]{\left\langle #1 \right\rangle}
\newcommand{\R}{\mathbb{R}}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {\bf CS 388R: Randomized Algorithms } \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[4]{\handout{#1}{#2}{#3}{Scribe: #4}{Lecture #1}}

\newtheorem{theorem}{Theorem}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{observation}[theorem]{Observation}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{assumption}[theorem]{Assumption}

% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex

\usepackage{hyperref}
\hypersetup{
    colorlinks=true,
    linkcolor=blue,
    filecolor=magenta,
    urlcolor=cyan,
}

\usepackage{filecontents}

\begin{filecontents}{references.bib}
  @book{BLM2013Concentration,
  Author = {St\'ephane Boucheron and G\'abor Lugosi and Pascal Massart},
  Date-Added = {2013-06-12 15:44:48 +0000},
  Date-Modified = {2013-06-12 15:47:32 +0000},
  Publisher = {Oxford University Press},
  Title = {Concentration Inequalities: A Nonasymptotic Theory of Independence},
  url = {http://opac.inria.fr/record=b1134861},
  Year = {2013}}

  @article{DBLP:journals/corr/abs-1011-3027,
  author    = {Roman Vershynin},
  title     = {Introduction to the non-asymptotic analysis of random matrices},
  journal   = {CoRR},
  volume    = {abs/1011.3027},
  year      = {2010},
  url       = {http://arxiv.org/abs/1011.3027},
  timestamp = {Mon, 05 Dec 2011 18:05:29 +0100},
  biburl    = {http://dblp.uni-trier.de/rec/bib/journals/corr/abs-1011-3027},
  bibsource = {dblp computer science bibliography, http://dblp.org}
  }
}
\end{filecontents}

\begin{document}

\lecture{6 --- Sep 16, 2015}{Fall 2015}{Prof.\ Eric Price}{Rishab Goyal}

\section{Overview}

In the last lecture we introduced subgaussian, subexponential and subgamma distributions, and also discussed few equivalent sufficient conditions for each of them.

In this lecture we first recall those definitions. Then we discuss Johnson-Lindenstrauss (JL) lemma and provide a proof for the same. After that we revisit the problem of coupon collector, and finally we provide a Bernstein-type inequality for a bounded range random variable.


\section{Preliminaries}

\begin{definition} (Subexponential random variable)
\label{sub-exp}
A random variable $X$ is said to be subexponential with parameter $\sigma$ iff $X - \E[X]$ satisfies any of the following three properties:
\begin{align*}
&\Pr\left[|X| \geq t\right] \leq 2 e^{-t/ (2 \sigma)} \quad \forall \ t > 0 \text{, or} \\
&\E\left[e^{\lambda X}\right] \leq e^{\lambda^2 \sigma^2 / 2} \quad \forall \ |\lambda| < 1 / \sigma \text{, or} \\
&\E\left[|X|^k\right]^{1/k} \leq \sigma \cdot k.
\end{align*}
If properties $(1)$ and $(3)$ hold for X, then they also hold for $X - \E[X]$, so $X$ is subexponential.\\
\end{definition}

\begin{definition} (Subgamma random variable)
\label{sub-gam}
A random variable $X$ is said to be subgamma with parameters $\sigma$ and $B$ if $X - \E[X]$ satisfies the following:
\begin{equation*}
\E\left[e^{\lambda X}\right] \leq e^{\lambda^2 \sigma^2 / 2} \quad \forall \ |\lambda| < B.
\end{equation*}
This additionally implies that:
\begin{equation*}
\Pr\left[|X| \geq t\right] \leq 2 \cdot \max\left(e^{-t^2/(2\sigma^2)}, e^{-Bt/2}\right).
\end{equation*}
\end{definition}

\vspace{4mm}

\begin{lemma}
\label{sum-sub-gam}
If $X_1, X_2, \ldots, X_n$ are independently distributed subgamma variables with parameters $(\sigma_i, B_i)$ for $i \in [n]$, then $X = \sum_{i = 1}^{n} X_i$ is also subgamma with parameters $(\sqrt{\sum_{i = 1}^{n} \sigma_i^2}, \min_{i \in [n]} B_i)$.
\end{lemma}

\begin{lemma}
\label{sum-normal}
If $X_1, X_2, \ldots, X_n$ are independent standard normal variables ($\sim N(0, 1)$), then $X^2$ is subgamma with parameters $(\sqrt{n}, 1)$, where $X^2 = \sum_{i = 1}^{n} X_i^2$.
\end{lemma}

\begin{proof}
Since $X_i \sim N(0, 1)$, we know that $\Pr\big[|X_i| \geq t\big] \leq e^{-t^2/2}$ (for all $t > 0$). This implies that $\Pr\big[X_i^2 \geq t\big] \leq e^{-t/2}$ (for all $t > 0$). Therefore, each $X_i^2$ is a subexponential with parameter $1\ (= O(1))$, and $X_i^2$ is subgamma with parameters $(1, 1)$ (as per Defintion~\ref{sub-gam}).

Now, using Lemma~\ref{sum-sub-gam}, we can conclude that $X^2$ is subgamma with parameters $(\sqrt{n}, 1)$.
\end{proof}


\section{Johnson-Lindenstrauss Lemma}

During lecture, the following question was posed which captures the essence of JL lemma.

\paragraph{Question.} Let $X_1, X_2, \ldots, X_n \in \R^d$. Find $Y_1, Y_2, \ldots, Y_n \in \R^m$ such that for all $i, j$:
\begin{align*}
(1 - \eps)\| x_i - x_j \|_2 \leq \| y_i - y_j \|_2 \leq (1 + \eps)\| x_i - x_j \|_2.
\end{align*}
How large should $m$ be?

\paragraph{} For completeness, we state the formal statement of the JL lemma:

\begin{theorem} (JL Lemma)
\label{JL}
If $X \subseteq \R^d$ such that $|X| = n$, then for every $0 < \eps < 1/2$, there exists a linear map $A : \R^d \rightarrow \R^m$ such that for all $x_i, x_j \in X$:
\begin{align*}
(1 - \eps)\| x_i - x_j \|_2 \leq \| Ax_i - Ax_j \|_2 \leq (1 + \eps)\| x_i - x_j \|_2
\end{align*}
and $m = \Theta\left(\dfrac{\log n}{\eps^2}\right)$.
\end{theorem}

\begin{proof}
Consider the following linear map $A$ such that $A_{i,j} \sim N(0, 1/m)$.\\\vspace{-10mm}
\paragraph{} Let $x \in \R^d$ and $y = Ax$. Observe that $(Ax)_i =
\sum_{j = 1}^{d} A_{i, j} x^{(j)}$. This is a sum of independent
gaussians, and since the sum of gaussians is itself a gaussian with
variance equal to the sum of individual variances, we have that
$(Ax)_i \sim N\Big(0, \sum_{j = 1}^{d} (x^{(j)})^2\Big)$. Therefore,
$(Ax)_i \sim N(0, \| x \|^2/m)$.\\\vspace{-8mm}
\paragraph{} Next, observe that $\| Ax \|^2 = \sum_{i = 1}^{m} (Ax)_i^2$. Therefore, $\E\big[ \| Ax \|^2 \big] = \sum_{i = 1}^{m} \E\big[(Ax)_i^2\big]$, using linearity of expectation. Since, $(Ax)_i \sim N(0, \| x \|^2/m)$, this implies that $\E\big[(Ax)_i^2\big] = Var((Ax)_i) = \| x \|^2/m$. So, $\E\big[ \| Ax \|^2 \big] = \| x \|^2$.\\\vspace{-8mm}
\paragraph{} Let $z_i = \dfrac{\sqrt{m}}{\|x\|} \cdot (Ax)_i$ for $i \in [m]$, and $\| z \|^2 = \sum_{i = 1}^{m} z_i^2 = \dfrac{m}{\|x\|^2} \cdot\|Ax\|^2$. It is straightforward to verify that $z_i \sim N(0, 1)$, and $z_i$'s are all independent. Therefore, using Lemma \ref{sum-normal}, we can claim that $\| z \|^2$ is subgamma with parameters $(\sqrt{m}, 1)$. Since $\| z \|^2$ is subgamma, we could write the following (as implied by Definition~\ref{sub-gam}):
\begin{equation*}
\Pr\Big[\big| \| z \|^2 - m \big| \geq t\Big] \leq 2 \cdot \max(e^{-t^2/(2m)}, e^{-t/2}).
\end{equation*}
Substituting $t$ with $2\eps m$, we get
\begin{equation*}
\Pr\Big[ \| z \|^2 \geq (1 + 2\eps) m \vee \| z \|^2 \leq (1 - 2\eps) m \Big] \leq 2 \cdot \max(e^{-2\eps^2 m}, e^{-\eps m}).
\end{equation*}
Since $0 < \eps < 1/2$, we can write that $\max(e^{-2\eps^2 m}, e^{-\eps m}) = e^{-2\eps^2 m}$. Also, $(1 + 2\eps)^{1/2} \geq (1 + \eps)$ and $(1 - 2\eps)^{1/2} \leq (1 - \eps)$, so we could rewrite the above inequality as:
\begin{equation*}
\Pr\Big[ \| z \| \geq (1 + \eps) \sqrt{m} \vee \| z \| \leq (1 - \eps) \sqrt{m} \Big] \leq 2 e^{-2\eps^2 m}.
\end{equation*}
Substituting $\|z\|$ with $\dfrac{\sqrt{m}}{\|x\|} \cdot\|Ax\|$, we get
\begin{equation*}
\Pr\Big[ \| Ax \| \geq (1 + \eps) \|x\| \vee \| Ax \| \leq (1 - \eps) \|x\| \Big] \leq 2 e^{-2\eps^2 m}.
\end{equation*}
Note that above inequality holds for all $x \in \R^d$, therefore substituting $x$ with $x_i - x_j$, and $Ax$ with $A(x_i - x_j) = y_i - y_j$, we get for all $x_i, x_j$
\begin{equation*}
\Pr\Big[ \| y_i - y_j \| \notin \big( (1 - \eps) \|x_i - x_j\|, (1 + \eps) \|x_i - x_j\| \big) \Big] \leq 2 e^{-2\eps^2 m}.
\end{equation*}
Therefore, using union bound, we can say that
\begin{equation*}
\Pr\Big[ A \text{ preserves distance between all pairs} \Big] \geq 1 - n^2 e^{-2\eps^2 m}.
\end{equation*}
So, for $m = c \dfrac{\log n}{\eps^2}$,
\begin{equation*}
\Pr\Big[ A \text{ preserves distance between all pairs} \Big] \geq 1 - \dfrac{1}{n^{2c - 2}} \geq 0.
\end{equation*}
This concludes the proof as we have proved that there exists a linear map $A$ which reduces the dimensionality of data and preserves its structure (distances) \emph{with high probability}.
\end{proof}


\section{Coupon Collector Problem}

In the $n$ coupon collector problem, $T_i$ denoted the number of draws to get the $i^{th}$ new coupon, and $T = \sum_{i = 1}^{n} T_i$ denoted the total number of draws made to draw each coupon at least once.

Recall that $T_i$'s are all independent of each other, and $T_i \sim Geom\Big(\dfrac{n + 1 - i}{n}\Big)$, $\E[T] = n H_n$, and $Var(T) = O(n^2)$. Using Markov's inequality for concentration, we get 
\begin{equation*}
\Pr\Big[ T \geq \dfrac{n H_n}{\delta} \Big] \leq \delta.
\end{equation*}
Similarly, Chebyshev gives 
\begin{equation*}
\Pr\Big[ T \geq n H_n +  \dfrac{O(n)}{\sqrt{\delta}} \Big] \leq \delta.
\end{equation*}
Now, we will try to get similar bounds using results from subgamma distributions. First, note that 
\begin{equation*}
\Pr[ T_i \geq t ] \leq \Big(1 - \dfrac{n + 1 - i}{n}\Big)^t \leq e^{-(\frac{n + 1 - i}{n})t}.
\end{equation*}
Therefore, $T_i$ is subexponential with parameter $\sigma_i = \dfrac{n}{n + 1 - i}$. Hence, $T$ is subgamma with mean $\mu = n H_n$ and parameters $(n \sqrt{\sum_{i = 1}^{n} \big(\dfrac{1}{n + 1 - i}\big)^2}, min_i\ \sigma_i^{-1}) = (O(n), \dfrac{1}{n})$. Since $T$ is subgamma, we could write the following:
\begin{equation*}
\Pr[T \geq \mu + t] \leq 2 \cdot \max\big(e^{-t^2/ (2 n^2)}, e^{-t / (2n)}\big) \leq 2 e^{-t / (2n)}.
\end{equation*}
Substituting $t$ with $O(n \log(1 / \delta))$, we get
\begin{equation*}
\Pr\Big[ T \geq n H_n +  O(n \log(1 / \delta)) \Big] \leq \delta.
\end{equation*}

\paragraph{} Note that all the above analysis was done using different concentration inequalities. Below we discuss how to obtain the same bound as obtained with subgamma analysis of $T$ using only union bound.

Let $F_{i, T}$ denote the event that coupon $i$ is not found after $T$ steps. Therefore, we can say that
\begin{equation*}
\Pr[F_{i, T}] = \Big( 1 - \dfrac{1}{n} \Big)^T \leq e^{-T/n}.
\end{equation*}
Substituting $T$ with $n \log (n/\delta)$, we get (for all $i \in [n]$)
\begin{equation*}
\Pr[F_{i, n \log (n/\delta)}] \leq \dfrac{\delta}{n}.
\end{equation*}
So, using union bound, we can conclude that
\begin{equation*}
\Pr[\text{Any coupon not found in } n \log (n/\delta) \text{ steps}] = \Pr\Big[ \bigvee\limits_{i \in [n]} F_{i, n \log (n/\delta)} \Big] \leq \delta.
\end{equation*}

\paragraph{} The above analysis gives an upper bound on the probability that all coupons are not found after $T$ steps. It should be interesting to find whether this bound is tight, or if we could do better. For that purpose, we start with lower bounding the probability of event $F_{i, T}$ as
\begin{equation*}
\Pr[F_{i, T}] = \Big( 1 - \dfrac{1}{n} \Big)^T \geq e^{- \frac{T}{n} (1 - O(\frac{1}{n}))} \geq \frac{\delta}{n} \Big( 1 - O\big(\frac{\delta}{n}\big) \Big),
\end{equation*}
where we substituted $T$ with $n \log (n/\delta)$. If we assume that the events $\neg F_{i, T}$ (coupon $i$ is found after $T$ steps) are \emph{independent}, then using the above inequality we could write the following
\begin{align*}
\Pr[\text{Any coupon not found in } n \log (n/\delta) \text{ steps}] &= 1 - \Pr[\text{All coupons found after } n \log (n/\delta) \text{ steps}]\\
                                                                     &= 1 - \Pr\Big[ \bigwedge\limits_{i \in [n]} \neg F_{i, n \log (n/\delta)} \Big]\\
                                                                     &\geq 1 - \Big( 1 -  \frac{\delta}{n} \Big( 1 - O\big(\frac{\delta}{n}\big) \Big) \Big)^n \approx \delta
\end{align*}
This would suggest that our bound is tight. But it assumes independence of events $\neg F_{i, T}$ which is incorrect, so we could only consider the above lower bound as a mild indicator of tightness. However, despite lack of independence, we can expect much better concentration because intuitively if some subset $I \subset [n]$ is drawn within first $T$ draws, then the event that some coupon $j \notin I$ is also drawn in first $T$ draws is less likely. Looking ahead, the key fact will be that $\neg F_{i ,T}$ are negatively correlated.

\paragraph{Next class.} First, observe that $\E[\text{Number of coupons not found after } n \text{ steps}] = n \Big( 1 - \dfrac{1}{n} \Big)^n \approx n / e$. In the next lecture, we will try to bound the probability that more than $n / 2$ coupons not found after $n$ draws ($\Pr[\text{Number of coupons not found after } n \text{ steps} \geq n/2]$).


\section{Bernstein-Type Inequality}

Recall that while discussing Chernoff bound, we observed that its main limitation is that it does not depend on the variance of each individual $X_i$. In the same vein, we try to improve the concentration bound for bounded range random variables. Below we prove that each bounded range variable is also subgamma. Since, sum of subgamma variables is also subgamma (from Lemma~\ref{sum-sub-gam}), therefore it has interesting concentration properties. Formally, we state the theorem below.\\

\begin{theorem}
\label{bernstein}
If $X$ is a random variable with range $[0, 1]$ and variance $s^2$, then it is also subgamma with parameters $(\sqrt{2} s, 1/2)$.
\end{theorem}

\begin{proof}
Let $Y = X - \E[X]$. This implies that $\E[Y^2] = Var(X) = s^2$ and $|Y| \leq 1$. Since $|Y| \leq 1$, we can also conclude that $\E\left[|Y|^k\right] \leq s^2$ for all $k > 2$. Next we will try to bound the moment generating function of $Y$.
\begin{align*}
\E[e^{\lambda Y}] &= \E\left[ \sum\limits_{k = 0}^{\infty} \dfrac{\lambda^k Y^k}{k!} \right]\\
                  &\leq 1 + \sum\limits_{k = 2}^{\infty} \dfrac{\lambda^k}{k!} s^2\\
                  &\leq 1 + \left( \sum\limits_{k = 0}^{\infty} \lambda^k \right) \dfrac{\lambda^2 s^2}{2}
\end{align*}
Note that the geometric series on the right converges only if $\lambda < 1$. So, for $\lambda < 1$, we could write
\begin{align*}
\E[e^{\lambda Y}] \leq 1 + \dfrac{\lambda^2 s^2}{2 (1 - \lambda)} \leq e^{(\lambda^2 s^2)/(2 (1 - \lambda))}.
\end{align*}
Now if we choose $\lambda$ such that $\dfrac{\lambda^2}{(1 - \lambda)} \leq 2 \lambda^2$, then the exponent in the previous bound would agree with that of a subgamma variable. Concretely, if we keep $\lambda \leq 1/2$, then we could write the following
\begin{align*}
\E[e^{\lambda Y}] \leq e^{(2 \lambda^2 s^2)/2} = e^{\lambda^2 (\sqrt{2} s)^2/2}.
\end{align*}
Therefore, $X$ is subgamma with parameters $(\sqrt{2} s, 1/2)$. This concludes the proof of Theorem~\ref{bernstein}.
\end{proof}

\paragraph{Biased coin toss.} Let $X_1, \ldots, X_n$ be $n$ i.i.d. Bernoulli random variables ($X_i \sim Ber(p)$), and $X = \sum_{i = 1}^{n} X_i \quad (\sim Bin(n, p))$. Applying Theorem~\ref{bernstein}, we get that each $X_i$ is subgamma with parameters $(\leq \sqrt{2p}, 1/2)$. Applying Lemma~\ref{sum-sub-gam}, we get that $X$ is also subgamma with parameters $(\sqrt{2pn}, 1/2)$. Therefore, 
\begin{align*}
\Pr[X \geq np + t] \leq 2 \cdot \max(e^{-t^2/(4np)}, e^{-t/4}).
\end{align*}
If $t \leq np$, then the controlling term would be $e^{-t^2/(4np)}$, else it would be $e^{-t/4}$. Therefore, $\Pr[X \geq 2np] \leq e^{-np/4}$ and $\Pr[X \geq np + t] \leq e^{-t^2/(4np)}$ for $t \in [0, np]$.

\nocite{*}

\bibliographystyle{abbrv}
\bibliography{references}

\end{document}