\documentclass[11pt]{article}
\usepackage{amsmath,amssymb,amsthm}
\usepackage{xcolor}

\DeclareMathOperator*{\E}{\mathbb{E}}
\let\Pr\relax
\DeclareMathOperator*{\Pr}{\mathbb{P}}

\newcommand{\eps}{\epsilon}
\newcommand{\inprod}[1]{\left\langle #1 \right\rangle}
\newcommand{\R}{\mathbb{R}}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {\bf CS 388R: Randomized Algorithms } \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
      \textcolor{red}{\textbf{NOTE:} THESE NOTES HAVE NOT BEEN EDITED OR CHECKED FOR CORRECTNESS}
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[4]{\handout{#1}{#2}{#3}{Scribe: #4}{Lecture #1}}

\newtheorem{theorem}{Theorem}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{observation}[theorem]{Observation}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{assumption}[theorem]{Assumption}

% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex

\begin{document}

\lecture{16 --- October 26, 2017}{Fall 2017}{Prof.\ Eric Price}{Kiyeon Jeon, Xiong Zheng }

\section{Overview}

In the last lecture we intoruced Johnson Lindenstrauss lemma and covered subgaussian random variable.

In this lecture we will cover subexponential and subgamma random variables.

\section{Revisit Coupon Collector problem}
Let's recall the coupon collector problem where we collect coupons until we find all $n$ types of coupons. Each one that arrives is uniform on $[n]$. Let $T$ be the total number you collect. In the lecture 5, we can know $T=nH_n+O(n)$ by Chebyshev inequality with mean $\E[T]$ and variance $\text{Var}[T]$. Now, we will deal with better concentration inequality with subexponential and subgamma random variables.\\
Let $T_i$ be the time to catch the $i$--th coupon. Then $T_i\sim\text{geom}(1-\frac{i-1}{n})$. $\mathbb{E}[T_i]=\frac{1}{1-\frac{i-1}{n}}=\frac{n}{n-i+1}$ $\mathbb{E}[T]=\sum \mathbb{E}[T_i]=nH_n=\Theta(n \log n)$. How can we know $\text{Pr} (T\geq nH_n+t)$ ?

When $x\sim\text{geom}(p)$, $\Pr[x=t]=p(1-p)^{t-1}$ $\Pr[x\geq t]=(1-p)^{t-1}\approx e^{-pt}\neq e^{-\frac{t^2}{2\sigma^2}}$ for any $\sigma$,  which means $x$ is not subgaussian variable. So, we cannot use concentration inequality from subgaussian.

\section{Concentration Inequality}
\subsection{Subgaussian}
A variable $X$ of mean $\mu$ is subgaussian with parameter $\sigma$ if
\begin{enumerate}
  \item {\em MGF} -- $\mathbb{E}[e^{\lambda(X-\mu)}]\leq e^{\frac{\lambda^2 \sigma^2}{2}}$ for all $\lambda\in\mathbb{R}$
  \item {\em Tail} -- $\Pr[|X-\mu|>t]\leq 2e^{-t^2/2\sigma^2}$ for all $t>0$
  \item {\em Moment} -- $\E[|X-\mu|^k]\leq k^{k/2}\sigma^k$ for all $k>0$
\end{enumerate}
The above three items are equivalent up to constant factors in $\sigma$

\subsection{Subexponential}
A variable $X$ of mean $\mu$ is subexponential with parameter $\sigma$ if
\begin{enumerate}
  \item {\em MGF} -- $\E[e^{\lambda(X-\mu)}]\leq e^{\frac{\lambda^2 \sigma^2}{2}}$ for all $|\lambda|<1/\sigma$
  \item {\em Tail} -- $\Pr[|X-\mu|>t]\leq 2e^{-t/2\sigma}$ for all $t$
  \item {\em Moment} -- $\E[|X-\mu|^k]\leq k^k\sigma^k$ for all $k$
\end{enumerate}
The above three items are equivalent up to constant factors in $\sigma$
\subsubsection{What MGF?}
Suppose $p(z)=e^{-z}$ for all $z>0$. $\Pr [Z>t]=e^{-t}$ and $\E [Z]=1$ Then, MGF is
\begin{align*}
E [e^{\lambda(Z-1)}]
&=\int^{\infty}_{0}e^{-z}e^{\lambda(z-1)} dz =\frac{e^{-\lambda}}{1-\lambda} \quad\text{ for } \lambda<1\\
&=\frac{1-\lambda+\frac{\lambda^2}{2}-\frac{\lambda^3}{3!}+\cdots}{1-\lambda}\\
&=1+\frac{\lambda^2}{2}+\lambda^3(\frac{1}{2}-\frac{1}{3!})+\lambda^4(\cdots)+\cdots \leq e^{4\lambda^2/2} \text{ for all } \,|\lambda|<\frac{1}{2}
\end{align*}
There is a problem that the subexponential is not closed under adding independent copies.
\subsection{Subgamma}
A variable $X$ of mean $\mu$ is subgamma with parameters $(\sigma,c)$ if:
\begin{enumerate}
  \item{\em MGF} -- $\E [e^{\lambda(X-\mu)}] \leq e^{ \frac{\lambda^2 \sigma^2}{2}}$ for all $|\lambda|<\frac{1}{c}$
  \item{\em Tail} -- $\Pr[|X-\mu|>t]\leq 2\max\{e^{-t^2/2\sigma^2}, 2 e^{-t/2c}\}$  for all $t$
\end{enumerate}
\begin{observation}
$\text{subexponential}(\sigma)=\text{subgamma}(\sigma,\sigma)$
\end{observation}
\begin{observation}
$\text{subgaussian}(\sigma)=\text{subgamma} (\sigma, 0^{+})$
\end{observation}
Tail means
\begin{align*}
\Pr [|X-\mu| >t]
&\leq \begin{cases} 2e^{-t^2/2\sigma^2},
\,&\text{if } \,|t|<\sigma^2/c \\
2e^{-t/2c},
\,&\text{if } |t|>\sigma^2/c \end{cases}
\end{align*}

It implies that with probability $1-\delta$, $|X-\mu|\leq \max\{ \sigma \sqrt{2\log(2/\delta)},2c\log(2/\delta) \}$
\begin{proposition} If $X$ and $Y$ are independent subgamma random variable i.e. $X\in \text{subgamma}(\sigma,c_1) $ and $Y\in \text{subgamma}(\sigma,c_2)$, then $X+Y\in \text{subgamma} (\sqrt{\sigma_1^2+\sigma_2^2},\max(c_1,c_2))$
\end{proposition}
\begin{proof}
for all $\lambda< \frac{1}{\max(c_1,c_2)}$, $\E [e^{\lambda(X+Y)}]=\E[e^{\lambda X}] \E[e^{\lambda Y}]\leq e^{\frac{\lambda^2 \sigma_1^2}{2}} e^{\frac{\lambda^2 \sigma_2^2}{2}}=e^{\frac{\lambda^2}{2} (\sqrt{\sigma_1^2+\sigma_2^2})^2}$
\end{proof}
\subsubsection{Coupon Collector problem}
Going back to the Coupon Collector problem,
\begin{align*}
&T_i\sim \text{geom}(1-\frac{i-1}{n}) \\
\Rightarrow&T_i\in \text{subexp}(\frac{n}{n-i+1})=\text{subgamma}(\frac{n}{n-i+1},\frac{n}{n-i+1})\\
\Rightarrow&\sum T_i \in \text{subgamma}(\Theta(n),\Theta(n)) (\because \sqrt{\sum_{i=1}^n (\frac{n}{n-i+1})^2} \approx \sqrt{n^2 \frac{\pi^2}{6}}  =\Theta(n))\\
\Rightarrow&\Pr [T\geq nH_n+t]\leq 2\cdot \max \{e^{-\frac{t^2}{2n^2}},e^{-\frac{t}{2n}}\}\leq 2\cdot e^{-\frac{t}{2n}}\\
\Rightarrow& T\leq nH_n+2n \log(2/\delta)=O(n\log n) \text{ with probability } 1-\delta
\end{align*}


\subsubsection{Bounded random variable}
Let $X_i\in [0,1]$ have variance $\sigma_i^2$. How about $X=\sum_{i=1}^{n} X_i$ with independent $X_i$'s?

\begin{lemma}
If random variable $X\in[0,1]$ has variance $\sigma$, then $X\in \text{subgamma}(\sigma\sqrt{2},2)$
\end{lemma}
\begin{proof}
Let $Y=X-\E[X]$
\begin{align*}
\E[e^{\lambda Y}]
=& \sum_{k=0}^{\infty} \frac{\E (\lambda Y)^k}{k!}\\
=& 1 + \sum_{k=2}^{\infty} \frac{\lambda ^k \E[Y^k]}{k!}\\
\overset{(a)}{\leq}& 1 + \sum_{k=2}^{\infty} \frac{\lambda ^k \sigma^2}{k!}\\
\leq& 1 + \frac{\lambda^2 \sigma^2}{2} \sum_{k=0}^{\infty} \lambda^k\\
=& 1 + \frac{\lambda^2 \sigma^2}{2(1-\lambda)}
\end{align*}
(a) is established by $|Y^k| \leq |Y^2| |Y^{k-2}|\leq |Y^2|$
\end{proof}

There are two methods for the concentration inequality regarding $\sum X_i$.
\begin{enumerate}
  \item{\em a naive way with subgaussian}
  \begin{align*}
  X_i   &\in \text{subgaussian}(\frac{1}{2})\\
  X     &\in \text{subgaussian}(\frac{\sqrt{n}}{2})\\
  X     &=\E[X] \pm \sqrt{2n\log(2/\delta)} \text{ w.p.} 1-\delta
  \end{align*}
  \item{\em a better way with subgamma }
  \begin{align*}
  X_i   &\in \text{subgamma}(\sqrt{2\sigma_i^2},2)\\
  X=\sum X_i   &\in \text{subgamma}(\sqrt{2\sum \sigma_i^2},2)\\
  \Pr [|X-\mu|\geq t] &\leq \max \{e^{-\frac{t^2}{4\sum \sigma_i^2}},e^{-\frac{t}{4}}\}=\max \{e^{-\frac{t^2}{4\text{Var}(X)}},e^{-\frac{t}{4}}\}\\
  X &=\E[X]\pm (2\sqrt{\text{Var}(X) \log(2/\delta)}+4\log(2/\delta)) \text{ w.p } 1-\delta
  \end{align*}
\end{enumerate}


\section{Proof of Distributional JL}
\begin{theorem}{(Distributional JL)}
There exists distributions on matrices $A\in\mathbb{R}^{m\times d}$ such that for any $x$ in $\mathbb{R}^d$ $\|Ax\|_2=(1\pm \eps) \|x\|_2$ with probability $1-\delta $ with $m=O(\frac{1}{\eps^2}\log(1/\delta))$
% Let $x\in \mathcal{R}^d$ with $\|x\|_2=1$. When we choose $A\in \mathcal{R}^{m\times d} $ where $A_{ij}\sim \mathcal{N}(0,1)$, $\Pr [ |\frac{\|Ax\|_2^2}{m}-\|x\|_2^2 |>\eps]\leq 1-\delta$ with $m=O(\frac{1}{\eps^2}\log(1/\delta))$
\end{theorem}
\begin{proof}
Let $x\in\mathbb{R}^d$ with $\|x\|_2=1$.
We can choose $A$ as $\mathcal{N}(0,1)^{m\times d}$ (or $\{-1,+1\}^{m\times d}$).
\begin{align*}
A_{ji} \in \text{subgaussian}(1)
&\Rightarrow A_{ji}x_{i} \in \text{subgaussian}(|x_i|)\\
&\Rightarrow (Ax)_j \in \text{subgaussian}(\sum |x_i|^2) = \text{subgaussian}(1)
\end{align*}
% We have $A_{ji} \in subgaussian(1)$, So $A_{ji}x_{i} \in subgaussian(|x_i|)$.
% Let $(Ax)_{j} = \sum A_{ji}x_{i}$. This is a sum of independent gaussians,  and  since  the  sum  of gaussians  is  itself  a  gaussian  with  variance  equal  to  the  sum  of individual variances. We can get $(Ax)_j \in subgaussian(\sum |x_i|^2) = subgaussian(1)$
Let $Z = (Ax)_j$
\begin{align*}
Z \in subgaussian(1)
&\Rightarrow \E[Z^2] = 1\\
&\Rightarrow \E[|Z^2|^k] = \E[|Z|^{2k}] \leq k^k\\
&\Rightarrow E[|Z^2-\mu|^k] \leq Z^kk^k\\
&\Rightarrow Z \in subexponential(\sigma=\Theta(1))\\
&\Rightarrow \sum(Ax)_j^2 \in subgamma(\sqrt{m}, 1)
\end{align*}

% Let $Z = (Ax)_j,Z \in subgaussian(1), ~ \E[Z^2] = 1$. Since $\E[|Z^2|^k] = \E[|Z|^{2k}] \leq k^k$, so $\E[|Z^2-\mu|^k] \leq Z^kk^k$, which implies $Z \in subexponential(\sigma=\Theta(1))$, i.e $(Ax)_j \in subexponential(\sigma=\Theta(1))$, so $\sum(Ax)_j^2 \in subgamma(\sqrt{m}, 1)  $,

Thus, we can get $\frac{\sum(Ax)_j^2}{m} = 1 \pm (\sqrt{\frac{2log(2/\delta)}{m}} + \frac{log(2/\delta)}{m})$ with probability $1-\delta$ if $m \geq \frac{\Theta(1)log(2/\delta)}{\epsilon^2}$
\end{proof}

% \newpage
% Here is an equation:
% \begin{equation*}
% a^2 + b^2 = c^2 ,
% \end{equation*}
% or
% $$
% a^2 + b^2 = c^2 .
% $$

% Here's the same equation but that can be referenced:
% \begin{equation}
% a^2 + b^2 + c^2 \label{eqn:pythagoras}
% \end{equation}
% Equation~\ref{eqn:pythagoras} has been known for a long
% time. Sometimes you might want to align equations:
% \begin{align*}
% (x+y)^2 - (x-y)^2 &= (x^2 + 2xy + y^2) - (x^2 - 2xy - y^2) \\
% {}&= 4xy
% \end{align*}
% or align them with numbers to be referenced:
% \begin{align}
% (x+y)^2 - (x-y)^2 &= (x^2 + 2xy + y^2) - (x^2 - 2xy - y^2) \label{eqn:1}\\
% {}&= 4xy \label{eqn:2}
% \end{align}
% or where only some of them can be easily referenced:
% \begin{align}
% \nonumber (x+y)^2 - (x-y)^2 &= (x^2 + 2xy + y^2) - (x^2 - 2xy - y^2) \\
% {}&= 4xy \label{eqn:3}
% \end{align}

% Here are some other math goodies: $\binom{n}{k}$, $\frac{n}{k}$, $x\gg
% y$, $x\ll y$, $x\lesssim y$, $x\gtrsim y$, $\inprod{x,y}$, $x\in\R^n$,
% $x\in S$, $S\ni x$, $x\notin S$, $x\neq y$, $x\oplus y$, $x\pm y$,
% $\sqrt{x}$, $x\equiv y$, $S\cap T$, $\S\cup T$, $S\subset T$,
% $S\subseteq T$, $S\supset T$, $\gamma$, $\Gamma$, $\R$, $\mathbb{F}$
% $$\sum_{i=1}^n i,\ \sum_{\substack{i=1\\i\ odd}}^n i, \prod_{i=1}^n i$$

% \section{Main Section}

% We begin by describing the problem \ldots.
% Make sure to use sections and subsections.

% \subsection{Blah blah blah}
% Here is a subsection.

% \subsubsection{Blah blah blah}
% Here is a subsubsection. You can use these as well.

% \subsection{Using Boldface}
% Make sure to use \textbf{lots} of {\bf boldface}.

% \paragraph{Question:}
% How would you use boldface?

% \paragraph{Example:}
% This is an example showing how to use boldface to help organize your
% lectures.


% \paragraph{Some Formatting.}
% Here is some formatting that you can use in your notes:
% \begin{itemize}
% \item {\em Item One} -- This is the first item.
% \item {\em Item Two} -- This is the second item.
% \item \dots and here are other items.
% \end{itemize}

% If you need to number things, you can use this style:
% \begin{enumerate}
% \item {\em Item One} -- Again, this is the first item.
% \item {\em Item Two} -- Again, this is the second item.
% \item \dots and here are other items.
% \end{enumerate}

% \paragraph{Bibliography.}
% Please give real bibliographical citations for the papers that we
% mention in class. See below for how to include a bibliography section.
% If you use BibTeX, integrate the .bbl file into your .tex source. You
% should reference papers like this: ``The tug of war sketch originates
% in a paper by Alon, Matias and Szegedy \cite{AMS99}.''  In general,
% the name of the authors should appear in text at most once (for the
% first citation); further citations look like: ``Our proof follows that
% of \cite{AMS99}''.

% Take a look at previous lectures (TeX files are available) to see the
% details. A excellent source for bibliographical citations is
% DBLP. Just Google DBLP and an author's name.



% \bibliographystyle{alpha}

% \begin{thebibliography}{42}

% \bibitem[AMS99]{AMS99}
% Noga~Alon, Yossi~Matias, Mario~Szegedy.
% \newblock The Space Complexity of Approximating the Frequency Moments.
% \newblock {\em J. Comput. Syst. Sci.}, 58(1):137--147, 1999.

% \end{thebibliography}

\end{document}
