\documentclass[11pt]{article}
\usepackage{amsmath,amssymb,amsthm}
\usepackage{xcolor}
\usepackage{hyperref}
\usepackage{enumerate}
\usepackage{mdframed}

\DeclareMathOperator*{\E}{\mathbb{E}}
\let\Pr\relax
\DeclareMathOperator*{\Pr}{\mathbb{P}}
\DeclareMathOperator{\normal}{\mathcal{N}}
\DeclareMathOperator{\Var}{Var}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {\bf CS 388R: Randomized Algorithms, Fall 2019 } \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
      \textcolor{red}{\textbf{NOTE:} THESE NOTES HAVE NOT BEEN EDITED OR CHECKED FOR CORRECTNESS}
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[5]{\handout{XX}{#3}{#4}{Scribe: #5}{Lecture #1: #2}}

\newtheorem{theorem}{Theorem}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{observation}[theorem]{Observation}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{assumption}[theorem]{Assumption}

% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex

\allowdisplaybreaks

%==========  FOR EDITING
\newcommand{\todo}[1]{{ \textcolor{magenta}{\textbf{(TO DO: }#1\textbf{)}} }}
\newcommand{\note}[1]{{ \textcolor{purple}{\textbf{(NOTE: }#1\textbf{)}} }}

\begin{document}

\lecture{17}{Subgamma variables and Johnson-Lindenstrauss}{October 24th, 2019}{Prof.\ Eric Price}{Sebastian Oberhoff}
% Example:
%\lecture{1}{Coupon Collector; Balls and Bins}{August 29th, 2019}{Prof.\ Eric Price}{John Doe, Jane Roe}

\section{Subgamma Variables}

\begin{definition}
  A random variable $X$ is \emph{subgamma} with variance proxy
  $\sigma^2$ and exponential scale $c$ if:
\[
\text{\emph{(I)}} \qquad \E\Bigl[e^{\lambda(X-\E[X])}\Bigr] \leq e^{\frac{\lambda^2\sigma^2}{2}} \qquad \forall \, \lvert \lambda \rvert \leq \frac{1}{c}
\]
\end{definition}

This definition implies:
\begin{enumerate}
\item[(II)]
\[
\Pr\bigl[\lvert X - \E[X]\rvert \geq t \bigr] \leq 2 \cdot \max\Bigl(e^{-\frac{t^2}{2\sigma^2}}, e^{-\frac{2}{2c}}\Bigr)\,.
\]
\item[(III)] With probability $\geq 1 - \delta$,
\[
\lvert X - \E[X] \rvert \leq \sqrt{2\sigma^2\ln\biggl(\frac{2}{\delta}\biggr)} + 2c\ln\biggl(\frac{2}{\delta}\biggr)\,.
\]
\end{enumerate}

Either of the latter two properties also implies the definition (I),
with a loss in parameters: $\sigma^2 \mapsto O(\sigma^2 + c^2)$ and
$c \mapsto O(c)$.

This is a generalization of the subgaussian random variables we
considered in last class, with the introduction of $c$.  In
particular, Subgaussian$(\sigma^2)$ = Subgamma$(\sigma^2, 0)$.

\subsection*{Example}

Let $Z \sim \normal(0,1)$ and $X = Z^2$.

Then
\[
\E[X] = \frac{1}{\sqrt{2\pi}} \int_{-\infty}^\infty z^2 e^{-\frac{z^2}{2}}\,dz = 1\,. \qquad \substack{\text{\footnotesize (split the integrand into $z \cdot ze^{-z^2/2}$} \\ \text{\footnotesize and use integration by parts)}}
\]
And the centered MGF is
\begin{align*}
\E\Bigl[e^{\lambda(X-1)}\Bigr] & = \frac{1}{\sqrt{2\pi}} \int_{-\infty}^\infty e^{\lambda(z^2-1)} e^{-\frac{z^2}{2}}\,dz\\
& = \frac{e^{-\lambda}}{\sqrt{2\pi}} \int_{-\infty}^\infty e^{\bigl(\lambda - \frac{1}{2}\bigr)z^2}\,dz\\
& = e^{-\lambda}\sigma \frac{1}{\sqrt{2\pi\sigma^2}} \int_{-\infty}^\infty e^{-\frac{z^2}{2\sigma^2}}\,dz & & \biggl(\lambda - \frac{1}{2} = -\frac{1}{2\sigma^2}\biggr)\\
& = e^{-\lambda} \sigma\\
& = \frac{e^{-\lambda}}{\sqrt{1-2\lambda}}\\
& \leq e^{\frac{4\lambda^2}{2}}\,. & & \biggl(\text{if }\lvert \lambda \rvert \leq \frac{1}{4}\biggr)\\
\end{align*}
So $X$ is Subgamma$(4,4)$.

\bigskip

Alternatively, if $Z$ is mean $0$ and Subgaussian$(\sigma^2)$ and $X = Z^2$ then we claim that $X$ is
\[
\text{Subgamma}\bigl(O(\sigma^4), O(\sigma^2)\bigr)\,.
\]
To show this note that from the definition of Subgaussian random variables we know that with probability $1 - \delta$
\[
Z = O\biggl(\sigma\sqrt{\log\Bigl(\frac{2}{\delta}\Bigr)}\biggr)\,.
\]
So by squaring both sides, we have
\[
Z^2 = O\biggl(\sigma^2 \log\Bigl(\frac{2}{\delta}\Bigr)\biggr)
\]
which fits (III) if $\sigma^2 = 0$ and $c = O(\sigma^2)$. Thus, after conversion we end up with variance proxy $O(\sigma^4)$ and exponential scale $O(\sigma^2)$.

\section{Basic Properties}

\begin{itemize}
\item If $X_1, X_2$ are independent and are $(\sigma_1^2, c_1)$- and $(\sigma_2^2, c_2)$-Subgamma, then $X_1 + X_2$ is $\bigl(\sigma_1^2 + \sigma_2^2, \max(c_1, c_2)\bigr)$-Subgamma.
\item If $X$ is $(\sigma^2, c)$-Subgamma, then $\alpha X$ is $(\alpha^2\sigma^2, \alpha c)$-Subgamma for any constant $\alpha$.
\end{itemize}

\section{Johnson-Lindenstrauss-Lemma (84)}

\begin{theorem}[Johnson-Lindenstrauss-Lemma]
Let $X_1, \dots, X_n \in \mathbb{R}^d$. Then there exist $y_1,\dots,y_n \in \mathbb{R}^m$ ($m$ ``small'') such that for all $i, j$:
\[
\lVert y_i - y_j \rVert_2 = (1 \pm \epsilon)\lVert x_i - x_j \rVert_2
\]
\end{theorem}

\begin{theorem}[Distributional Johnson-Lindenstrauss-Lemma]
There exists a random linear map $A \in \mathbb{R}^{m\times d}$ (entries of $A \sim \normal(0, 1/m)$) such that $\forall x \in \mathbb{R}^d$
\[
\lVert Ax \rVert_2 = (1 \pm \epsilon) \lVert x \rVert_2 \qquad \text{with probability } 1 - 2e^{-\Omega(\epsilon^2 m)}
\]
(or with probability $1-\delta$ if $m = O\bigl(\frac{1}{\epsilon^2}\log(\frac{2}{\delta})\bigr)$).
\end{theorem}

\subsection*{From The Distributional To the Standard Johnson-Lindenstrauss-Lemma}

Set $y_i = Ax_i$ with $\delta = \frac{1}{n^3}$. Then with probability $1 - \frac{1}{n}$ we have for all $i,j$:
\[
\underbrace{\lVert A(x_i - x_j) \rVert}_{= \lVert y_i - y_j \rVert} = (1 \pm \epsilon) \lVert x_i - x_j \rVert
\]
(Probability $> 0$ certainly implies existence.)

\subsection*{Proving The Distributional Johnson-Lindenstrauss-Lemma}

\begin{proof}
Select the entries of $A$ according to $\normal(0, 1/m)$. Then, denoting the $i$th row of $A$ as $A_i$, we have
\[
\forall x: \qquad y_i = A_i x \sim \sum_{j=1}^m \normal(0,1/m) x_j = \normal\Bigl(0, \underbrace{\sum_{j=1}^m x_j^2}_{= \lVert x \rVert_2^2}/m\Bigr)
\]
meaning $y_i$ is normally distributed with mean $0$ and variance $\frac{1}{m} \lVert x \rVert_2^2$.

Therefore,
\[
\E\bigl[\lVert y \rVert_2^2\bigr] = \sum{i=1}^m \E[y_i^2] = m \cdot \Var[y_i] = \lVert x \rVert_2^2
\]

All we need then is
\[
\Pr\Bigl[\lVert y \rVert_2^2\text{ is far from }\E\bigl[\lVert y \rVert_2^2\bigr]\Bigr] \leq \text{ something small.}
\]

Suppose $y_i \sim \normal(0,1)$. How does $\sum_{i=1}^m y_i^2$ concentrate about the expected value $m$?

$\sum y_i^2$ is Subgamma$(4m, 4)$. Hence,
\begin{align*}
\Pr\biggl[\Bigl\lvert \sum y_i^2 - m \Bigr\rvert \geq t \biggr] & \leq 2 \cdot \max\Bigl(e^{-\frac{t^2}{8m}}, e^{-\frac{t}{8}}\Bigr)\\
& \leq 2 \cdot \max\Bigl(e^{-\frac{\epsilon^2 m}{8}}, e^{-\frac{\epsilon m}{8}}\Bigr) & & (t = \epsilon m)\\
& = 2 e^{-\frac{\epsilon^2 m}{8}}\,. & & \text{(if $\epsilon < 1$)}\\
\end{align*}
\end{proof}

\section{Bernstein-type Bound}

If $\lvert X - \mu \rvert \leq m$ with probability $1$---in other words, if $X$ is restricted to a finite interval---then $X$ is Subgamma$\bigl(2\cdot \Var[X], 2m\bigr)$.

\subsection*{Example 1}

$X_i$ is a coin with bias $p_i$ towards $0$. And $X = \sum_i X_i$. Then
\[
\E[X] = \sum_i p_i = \mu \quad \text{and} \quad \Var[X] \leq \mu\,.
\]

Therefore, $X$ is Subgamma$(2\mu, 2)$ which implies
\[
\Pr\bigl[\lvert X - \mu \rvert \geq \epsilon \mu \bigr] \leq 2 e^{-\min(\epsilon^2, \epsilon) \mu/4}\,. \qquad \text{(multiplicative Chernoff)}
\]

\subsection*{Example 2}

In the coupon collector problem we had
\[
T_i \sim \text{geom}(p_i) \qquad p_i = \frac{n+1-i}{n}
\]
where $T_i$ was the arrival time of the $i$th item.

Consequently,
\[
\E[T_i] = \frac{n+1-i}{n}
\]
and
\[
\E[T] = \sum_i \E[T_i] = n H_n = O(n\log(n))\,.
\]

By the Bernstein bound we have that $T_i$ is Subgamma$\Bigl(O\bigl(\frac{1}{p_i^2}\bigr), O\bigl(\frac{1}{p_i}\bigr)\Bigr)$. Therefore, $T = \sum T_i$ is
\[
\text{Subgamma}\Biggl(O\biggl(\sum\frac{1}{p_i^2}\biggr), O\biggl(\max \frac{1}{p_i}\biggr)\Biggr) = \text{Subgamma}(n^2, n)\,.
\]
Hence, with probability $1 - \delta$
\begin{align*}
T & \leq \E[T] + O\biggl(\sqrt{n^2\log\Bigl(\frac{2}{\delta}\Bigr)} + n\log\Bigl(\frac{1}{\delta}\Bigr)\biggr)\\
& = O\biggl(n \log\Bigl(\frac{n}{\delta}\Bigr)\biggr)\,.\\
\end{align*}

\end{document}