\documentclass[11pt]{article}
\usepackage{amsmath,amssymb,amsthm}
\usepackage{xcolor}

\DeclareMathOperator*{\E}{\mathbb{E}}
\let\Pr\relax
\DeclareMathOperator*{\Pr}{\mathbb{P}}

\newcommand{\eps}{\epsilon}
\newcommand{\inprod}[1]{\left\langle #1 \right\rangle}
\newcommand{\R}{\mathbb{R}}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {\bf CS 388R: Randomized Algorithms, Fall 2019 } \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
      \textcolor{red}{\textbf{NOTE:} THESE NOTES HAVE NOT BEEN EDITED OR CHECKED FOR CORRECTNESS}
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[5]{\handout{XX}{#3}{#4}{Scribe: #5}{Lecture #1: #2}}

\newtheorem{theorem}{Theorem}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{observation}[theorem]{Observation}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{assumption}[theorem]{Assumption}

% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex

\begin{document}

\lecture{16}{Concentration Inequalities}{October 22nd, 2019}{Prof.\ Eric Price}{Zihang He, Sabee Grewal}
% Example:
%\lecture{1}{Coupon Collector; Balls and Bins}{August 29th, 2019}{Prof.\ Eric Price}{John Doe, Jane Roe}

\section{Overview}

In the last lecture we learned some concentration inequalities\ldots.

\begin{itemize}
\item Markov's inequality: if $X\geq 0$, $\Pr[X\geq t]\leq \frac{E[X]}{t}$;
\item Chebyshev's inequality: Let $\mu=E[X]$, $\Pr[|X-\mu|\geq t]=\Pr[(X-\mu)^2\geq t^2]\leq \frac{E[(X-\mu)^2]}{t^2}$;
\item Higher moments: for any even $k$, $\Pr[(X-\mu)^k\geq t^k]\leq \frac{E[(X-\mu)^k]}{t^k}$.
\end{itemize}


In this lecture we will learn more about concentration inequalities and prove the
additive form of the Chernoff bound.

\section{Moment Generating Functions}

\subsection{Gaussian refresher}

Gaussian $\mathcal{N}(0,1)$: $p(x)=\frac{1}{\sqrt{2\pi}}e^{-x^2/2}$. Let 
$X \sim \mathcal{N}(0,1)$, then

$$E[X^2]=1,\qquad E[X^k]=\frac{k!}{2^{k/2}(k/2)!}=(\Theta(k))^{k/2}.$$

With Chebyshev's inequality, we get a bound of $\Pr[(X-\mu)^2 \geq t^2] \leq \frac{1}{t^2}$. With the inequality for the $k$-th moment (for even $k$), we get a bound of $c\frac{k^{k/2}}{t^k}.$
To get the tightest bound, we want to find an even $k$ which minimizes $\frac{k^{k/2}}{t^k}$ (where $t$ is known). Note that as $k \xrightarrow{} k+1$, the numerator grows by $\sim \sqrt{k}$ and the denominator grows by $\sim t$, so we want $k \approx t^2$.

\subsection{Moment generating functions}

The moment generating function (mgf) is defined as

$$\phi(\lambda):=E[e^{\lambda(X-\mu)}].$$

By Taylor expansion, the mgf can be written as

$$\phi(\lambda)=E[1+\lambda(X-\mu)+\frac{\lambda^2}{2}(X-\mu)^2+\frac{\lambda^3}{3!}(X-\mu)^3+\cdots+\frac{\lambda^k}{k!}(X-\mu)^k].$$

Note, by the linearity of expectation, we see that $\E[\lambda (X- \mu)] = 0$ and that 
term disappears from the Taylor expansion. Additionally, as $\lambda$ grows, 
$\phi(\lambda)$ grows 
by a weighted combination of all moments. For larger $\lambda$, $\phi(\lambda)$ will be weighted
more towards the larger moments. We get a new bound for $\Pr[X-\mu\geq t]$ by applying 
Markov's inequality to the mgf:
$$\Pr[X-\mu\geq t]=\Pr[e^{\lambda(X-\mu)}\geq e^{\lambda t}]\leq \min_{\lambda\geq 0}\frac{\phi(\lambda)}{e^{\lambda t}}.$$

This bound is true for all mgfs. Now we'll consider mgfs of Gaussian random variables. First, let's consider the mgf of $X\sim\mathcal{N}(0,1)$:

\begin{equation*}
\begin{split}
    \phi(\lambda)&=E[e^{\lambda(X-\mu)}]=\int_{-\infty}^\infty \frac{1}{\sqrt{2\pi}}e^{-\frac{x^2}{2}}e^{\lambda x}dx\\
    &=\int_{-\infty}^\infty \frac{1}{\sqrt{2\pi}}e^{-\frac{(x-\lambda)^2}{2}}e^{\frac{\lambda^2}{2}}dx\\
    &=e^{\frac{\lambda^2}{2}}\int_{-\infty}^\infty \frac{1}{\sqrt{2\pi}}e^{-\frac{(x-\lambda)^2}{2}}dx\qquad\qquad\qquad\left(\frac{1}{\sqrt{2\pi}}e^{-\frac{(x-\lambda)^2}{2}}\text{ is the pdf of }N(\lambda,1)\right)\\
    &=e^{\frac{\lambda^2}{2}}.
\end{split}
\end{equation*}

Now, let's consider the more general Gaussian distribution $X \sim\mathcal{N}(0,\sigma^2)$:

$$E[X^2]=\sigma^2,\qquad E[X^k]=(\Theta(k)\sigma^2)^{k/2}.$$

The mgf of $X \sim \mathcal{N}(0,\sigma^2)$ is

\begin{align*}
    \phi(\lambda)&=E[e^{\lambda(X-\mu)}]=\int_{-\infty}^\infty \frac{1}{\sqrt{2\pi\sigma^2}}e^{-\frac{x^2}{2\sigma^2}}e^{\lambda x}dx\\
    &=\int_{-\infty}^\infty \frac{1}{\sqrt{2\pi\sigma^2}}e^{-\frac{(x-\lambda\sigma^2)^2}{2\sigma^2}}e^{\frac{\lambda^2\sigma^2}{2}}dx\\
    &=e^{\frac{\lambda^2\sigma^2}{2}}\int_{-\infty}^\infty \frac{1}{\sqrt{2\pi\sigma^2}}e^{-\frac{(x-\lambda\sigma^2)^2}{2\sigma^2}}dx\qquad\qquad\qquad\left(\frac{1}{\sqrt{2\pi\sigma^2}}e^{-\frac{(x-\lambda\sigma^2)^2}{2}}\text{ is the pdf of }N(\lambda\sigma^2,\sigma^2)\right)\\
    &=e^{\frac{\lambda^2\sigma^2}{2}}.
\end{align*}

Applying this mgf to the bound we found above, we get:

$$\Pr[X-\mu\geq t]\leq \min_{\lambda\geq 0}\frac{\phi(\lambda)}{e^{\lambda t}}=\min_{\lambda\geq 0}e^{\frac{\lambda^2\sigma^2}{2}-\lambda t}=e^{-\frac{t^2}{2\sigma^2}}$$
\Big(By completing square, $\frac{\lambda^2\sigma^2}{2}-\lambda t=\frac{1}{2}(\lambda\sigma-\frac{t}{\sigma})^2-\frac{t^2}{2\sigma^2}$\Big)

\section{Subgaussian, Subexponential, and Subgamma Random Variables}

\subsection{Subgaussian random variables}

\begin{definition}
A random variable $X$ is \text{subgaussian} with "variance proxy" (a.k.a. "parameter") $\sigma^2$ if
\begin{enumerate}
\item $\forall \lambda, E[e^{\lambda(X-\mu)}]\leq e^{\frac{\lambda^2\sigma^2}{2}}$
\item $\Pr[|X-\mu|\geq t]\leq 2e^{-\frac{t^2}{2\sigma^2}}$
\item $E[|X-\mu|^k]\leq k^{\frac{k}{2}}\sigma^k$
\end{enumerate}
\end{definition}

All three definitions above are equivalent up to constant factors in $\sigma$.

\begin{lemma}
Any variable $X$ bounded in $[a,a+b]$ is subgaussian with variance proxy $(\frac{b}{2})^2$.
\end{lemma}

\begin{proof}
You will be asked to prove this on the problem set!
\end{proof}

\begin{lemma}
If $X_1,X_2$ are independent subgaussian with variance proxies  $\sigma_1^2,\sigma_2^2$, $X_1+X_2$ is subgaussian with variance proxy $\sigma_1^2+\sigma_2^2$.
\end{lemma}

\begin{proof}
Assume $\mu=0$.
$$E[e^{\lambda(X_1+X_2)}]=E[e^{\lambda(X_1)}e^{\lambda(X_2)}]=E[e^{\lambda(X_1)}]E[e^{\lambda(X_2)}]\leq e^{\frac{\lambda^2\sigma_1^2}{2}}e^{\frac{\lambda^2\sigma_2^2}{2}}=e^{\frac{\lambda^2(\sigma_1^2+\sigma_2^2)}{2}}$$
\end{proof}

Note, $X\in$ subgaussian$(\sigma^2) :=$ $X$ is subgaussian with variance proxy $\sigma^2$.

Let's consider coin flip example: sum of $n$ coin flips $x_i\in\{0,1\}. X_i\in$ subgaussian$(\frac{1}{4})$, so $\sum X_i\in$ subgaussian $(\frac{n}{4})$:
$$\Pr[\sum X_i\geq \mu+t]\leq e^{-\frac{2t^2}{n}}.$$

This gives us the additive Chernoff bound! 

\subsection{Subexponential random variables}

$Z_i=$~\# flips until heads, then
$$\Pr[Z_i=t]=\frac{1}{2^t}, \qquad \E[Z_i] = 2.$$

Question: how do we bound $\Pr[\sum_{i=1}^{n} Z_i\geq 2n+t]$?

First, we find the mgf of $Z_i$: 
$\phi(\lambda)=E[e^{\lambda (Z-2)}]=\sum_{i=1}^\infty\frac{e^{\lambda i}}{2^i}e^{-2\lambda}$

\begin{definition}
A random $X$ is subexponential with "parameter" $\sigma^2$ if
\begin{enumerate}
\item $\forall \lambda\text{ s.t. } |\lambda|\leq \frac{1}{\sigma},\qquad E[e^{\lambda(X-\mu)}]\leq e^{\frac{\lambda^2\sigma^2}{2}}$
\item $\Pr[|X-\mu|\geq t]\leq 2e^{-\frac{t}{2\sigma}}$
\item $E[|X-\mu|^k]\leq k^{k}\sigma^k$
\end{enumerate}
\end{definition}

Example of subexponentials:
\begin{itemize}
    \item $\Pr[i]=\frac{1}{2^i}$
    \item $p(x)=e^{-x}\forall x\geq 0$
    \item $X^2$ if $X$ is subgaussian.
\end{itemize}

\begin{definition}
A random $X$ is subgamma with "parameter" $(\sigma^2,c)$ if
\begin{enumerate}
\item $\forall \lambda\text{ s.t. } |\lambda|\leq \frac{1}{c},\qquad E[e^{\lambda(X-\mu)}]\leq e^{\frac{\lambda^2\sigma^2}{2}}$
\item $\Pr[|X-\mu|\geq t]\leq 2~\max(e^{-\frac{t^2}{2\sigma^2}},e^{-\frac{t}{2\sigma}})$
\end{enumerate}
\end{definition}

Next lecture we''ll continue to explore subgamma and subexponential random variables.

%\bibliographystyle{alpha}
%\begin{thebibliography}{42}
%\end{thebibliography}

\end{document}