\documentclass[11pt]{article}
\usepackage{amsmath,amssymb,amsthm}
\usepackage{xcolor}
\usepackage{mathtools}

\DeclareMathOperator*{\E}{\mathbb{E}}
\let\Pr\relax
\DeclareMathOperator*{\Pr}{\mathbb{P}}

\newcommand{\eps}{\epsilon}
\newcommand{\inprod}[1]{\left\langle #1 \right\rangle}
\newcommand{\R}{\mathbb{R}}

\newcommand{\norm}[1]{\left\lVert#1\right\rVert}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {\bf CS 388R: Randomized Algorithms } \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
      \textcolor{red}{\textbf{NOTE:} THESE NOTES HAVE NOT BEEN EDITED OR CHECKED FOR CORRECTNESS}
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[4]{\handout{#1}{#2}{#3}{Scribe: #4}{Lecture #1}}

\newtheorem{theorem}{Theorem}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{observation}[theorem]{Observation}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{assumption}[theorem]{Assumption}

\newcommand*\diff{\mathop{}\!\mathrm{d}}
\newcommand*\Diff[1]{\mathop{}\!\mathrm{d^#1}}

% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex

\begin{document}

\lecture{15 --- October 24, 2017}{Fall 2017}{Prof.\ Eric Price}{Naren Manoj, Ryan Rice}

\section{Overview}

In this lecture we state the Johnson Lindenstrauss Lemma and the Distributional Johnson Lindenstrauss Lemma. We then develop some of the probability machinery required to prove these, such as Moment Generating Functions, Subgaussian variables, and the derivation of the additive Chernoff bound. 

\section{JL Lemma}

Note: All norms are $\ell^2$ norms.

\paragraph{Johnson Lindenstrauss Lemma}

For any set of $n$ vectors $x_1, \dots, x_n \in \R^d$ there exists an embedding $y_1, \dots, y_n \in \R^m$ such that $$\forall i, j \in [n], \;\; \norm{y_i - y_j}_2 \in (1 \pm \epsilon)\norm{x_i - x_j}_2$$
As long as $m = O(\frac{1}{\epsilon^2} \log n)$. 

Notice that we can get an exact embedding ($\epsilon = 0$) in $n$ dimensions by projecting the vectors onto the subspace spanned by $x_1, \dots, x_n$. 

\paragraph{Distributional Johnson Lindenstrauss Lemma} 
If we fix $0 < \epsilon, \delta < 1/2$, then there exists distributions on matrices $A \in \R^{m \times d}$ such that for any $x \in \R^d$, $\norm{A x}_2 \in (1 \pm \epsilon) \norm{x}_2$ with probability $1 - \delta$ with $m = O(\frac{1}{\epsilon^2} \log \frac{1}{\delta})$. 

The Johnson Lindenstrauss Lemma follows from this because $$\norm{y_i - y_j}_2 = \norm{Ax_i - Ax_j}_2 = \norm{A(x_i - x_j)}_2 \in (1 \pm \epsilon) \norm{x_i - x_j}_2$$

\section{Probability Background}

Here, we'll develop some of the probability tools that are useful both in proving the JL Lemma and Chernoff bounds.

\subsection{Concentration Inequalities}

\subsubsection{Markov's Inequality}

Recall Markov's Inequality. For a nonnegative random variable $X$ and positive $t$, we have:

$$\Pr[X \ge t] \le \frac{\E[X]}{t}$$

\subsubsection{Chebyshev's Inequality and $k$th Moments}

We can use Markov's Inequality to derive Chebyshev's Inequality. Suppose $X$ has mean $0$. Then,

\begin{align*}
    \Pr[|X| \ge t] &= \Pr[X^2 \ge t^2] \\
                   &\le \frac{\E[X^2]}{t^2}
\end{align*}

Letting $X = Y - \E[Y]$ gives us the inequality we're familiar with. \\

Notice that this exponentiation trick works for even powers $k$:

\begin{align*} 
    \Pr[|X| \ge t] &= \Pr[X^k \ge t^k]\\
                   &\le \frac{\E[X^k]}{t^k}
\end{align*}

Equivalently, we can write these in terms of a failure probability $\delta$, where the failure is marked by some deviation from the mean. Concretely, we set $\Pr[|X| > t] = \delta$, which yields:

% We can also phrase this in terms of $\delta$, where with probability $1 - \delta$,

$$|x| \leq \left(\frac{\E[X^k]}{\delta}\right)^{1/k}$$

\subsubsection{Moment Bounds on the Sums of Random Variables}

Now, let's say that $X$ is the sum of several independent random variables $X_i \in [-1, 1]$ where $\E[X_i] = 0$. In particular, set:

$$X = \sum_{i = 1}^n X_i$$

We can try to come up with some tail bound for $X$ in terms of the number of observations we have ($n$). Using our previous result, we can say:

\begin{align*}
    \Pr[|X| \ge t] &\le \frac{\E[X^2]}{t^2}
\end{align*}

Note that it suffices to determine an upper bound for $\E[X^2]$ in terms of $n$. Doing so will allow us to substitute this bound into the above inequality. To do so, consider the below:

\begin{align*}
    \E[X^2] &= \E\left[\left(\sum_{i = 1}^n X_i \right)^2\right] \\
            &= \E\left[\sum_{i = 1}^n X_i^2 + \sum_{i \neq j} X_iX_j \right] \\
            &= \E\left[\sum_{i = 1}^n X_i^2\right] + \E\left[\sum_{i \neq j} X_iX_j \right] \\
            &= \sum_{i = 1}^n \E\left[X_i^2\right] + \sum_{i \neq j} \E\left[X_iX_j\right] \\
            &= \sum_{i = 1}^n \E\left[X_i^2\right] + \sum_{i \neq j} \E\left[X_i\right]\E\left[ X_j\right] \\
            &= \sum_{i = 1}^n \E\left[X_i^2\right] \\
            &\le \sum_{i = 1}^n 1 \\
            &= n
\end{align*}

Thus, we have:

$$\Pr[|X| \ge t] \le \frac{\E[X^2]}{t^2} \le \frac{n}{t^2}$$

% Now notice that for higher even powers, we can still expand and remove any terms where there is some $X_i$ with power $1$. So for any even $k$, we can say that

In fact, we can generalize the above result to arbitrary even integers $k$:
\begin{equation} \E\left[\left(\sum_{i = 1}^n X_i\right)^k\right] \le k^{k/2}n^{k/2} \label{eqn:moment_bound} 
\end{equation} 


Using \ref{eqn:moment_bound}, we have:

\begin{align*}
    \Pr[|X| \ge y] &\le \frac{\E\left[X^k\right]}{y^k} \\
                   &\le \frac{k^{k/2}n^{k/2}}{y^k}
\end{align*}

\subsubsection{Obtaining a Chernoff-like Bound}

Note that our previous inequality is true for all even $k$. As a result, we can select some $k$ which makes the bound as tight as possible. To do so, set $y = t\sqrt{n}$ below:

\begin{align*}
    \Pr[|X| \ge t\sqrt{n}] &\le \frac{k^{k/2}n^{k/2}}{t^kn^{k/2}} \\
                           &\le \left(\frac{\sqrt{k}}{t}\right)^k
\end{align*}

Now, we can use some calculus to determine the value of $k$ which minimizes the right hand side. Once we do this, we find that $k = t^2/e$. Substituting this back gives us:

\begin{align*}
    \Pr[|X| \ge t\sqrt{n}] &\le \left(\frac{\sqrt{t^2/e}}{t}\right)^{t^2/e} \\
                           &= \left(e^{-1/2}\right)^{t^2/e} \\
                           &= \exp\left(-\frac{t^2}{2e} \right)
\end{align*}

Now, set $u = t/\sqrt{n}$. This yields:

$$\Pr[|X| \ge u\sqrt{n}] = \Pr[|X| \ge t] \le \exp\left(-\frac{t^2}{2ne} \right) = \exp\left(-\Omega\left(\frac{t^2}{n}\right)\right)$$

which gives us something that looks like a Chernoff bound. 

\subsection{Moment Generating Functions and Subgaussian Variables}

Observe that our previous analysis is somewhat faulty. Our moment bound is only known to hold true for even $k$, but in our computations, we set $k = t^2/e$. This is clearly not always an integer (and much less an even integer). We can construct a more sound analysis, but in order to do so, we have to introduce a new tool.

\begin{definition}[Moment Generating Function]
The Moment Generating Function (MGF) of a random variable $X$ and parametrized by $\lambda \in \mathbb{R}$ is:
$$M_X(t) = \E\left[e^{\lambda (X - \E[X])}\right]$$
\end{definition}

To motivate the MGF, it helps to take a look at the below manipulations (for $X$ being mean $0$):

\begin{align*}
    \E\left[e^{\lambda X}\right] &= \E\left[1 + \lambda X + \frac{\lambda^2X^2}{2} + \frac{\lambda^3X^3}{6} + \dots + \frac{\lambda^rX^r}{r!} + \dots\right] \\
                                 &= \sum_{r = 0}^{\infty} \E\left[\frac{\lambda^rX^r}{r!}\right] \\
                                 &= \sum_{r = 0}^{\infty} \frac{\lambda^r}{r!}\E\left[ X^r\right] \\
\end{align*}

where we obtain the above from linearity of expectation and the Taylor Series expansion of $e^{\lambda X}$. Notice that this expansion results in each moment of $X$ having some nonzero coefficient. In other words, all moments are ``generated'' (hence the name of the function). Furthermore, notice that selecting a varying value of $\lambda$ results in different moments being prioritized. This is analogous to selecting some optimal value of $k$ above, and we can use very similar ideas to obtain our desired bounds.

\subsubsection{Moment Generating Function of a Gaussian}

Let's put our new machinery to use by computing the MGF of a distribution. \\

Quite often, we work with Gaussians. We also get relatively well-bounded tail behavior with a Gaussian; hence, computing the MGF of a Gaussian seems like it might be useful. Consider the below:

\begin{align*}
    M_X(\lambda) &= \int_{-\infty}^{\infty} p(t)e^{\lambda t} \diff t \\
           &= \int_{-\infty}^{\infty} \frac{1}{\sqrt{2\pi\sigma^2}} \cdot e^{-t^2/2\sigma^2} \cdot e^{\lambda t} \diff t \\
\end{align*}

To simplify things a bit, we can complete the square in the exponents and manipulate that result:

\begin{align*}
    &\ -\frac{t^2}{2\sigma^2} + \lambda t \\
    &= -\left(\frac{t^2}{2\sigma^2} - \lambda t \right) \\
    &= -\left(\left(\frac{t}{\sigma\sqrt{2}} - \frac{\sqrt{2}\sigma\lambda}{2} \right)^2 - \frac{\sigma^2\lambda^2}{2} \right) \\
    &= -\left(\frac{t}{\sigma\sqrt{2}} - \frac{\sqrt{2}\sigma\lambda}{2} \right)^2 + \frac{\sigma^2\lambda^2}{2} \\
    &= -\left(\frac{t}{\sigma\sqrt{2}} - \frac{\sigma^2\lambda}{\sigma\sqrt{2}} \right)^2 + \frac{\sigma^2\lambda^2}{2} \\
    &= -\frac{\left(t-\sigma^2\lambda \right)^2}{2\sigma^2} + \frac{\sigma^2\lambda^2}{2}
\end{align*}

Define $p'(t)$ below: 

$$p'(t) = \frac{1}{\sqrt{2\pi\sigma^2}} \cdot e^{-(t - \sigma^2\lambda)^2/2\sigma^2}$$

And note that we can rewrite our messy integral in terms of $p'(t)$:

$$\int_{-\infty}^{\infty} p'(t)e^{\sigma^2\lambda^2/2} \diff t$$

Observe that $p'(t)$ represents a Gaussian that is horizontally shifted. Furthermore, observe that $\exp(\sigma^2\lambda^2)$ is constant in $t$. We now have:

\begin{align*}
    &\ \int_{-\infty}^{\infty} p'(t)e^{\sigma^2\lambda^2/2} \diff t \\
    &= e^{\sigma^2\lambda^2}\int_{-\infty}^{\infty} p'(t) \diff t \\
    &= e^{\sigma^2\lambda^2/2}
\end{align*}

We therefore conclude that if $X$ is Gaussian with variance $\sigma^2$, then:

$$M_X(\lambda) = e^{\lambda^2\sigma^2/2}$$

\subsubsection{Subgaussian Random Variables}

Subgaussian random variables are, intuitively, random variables which yield better tail bounds than Gaussian variables. We can formally define them here:

\begin{definition}
A random variable with mean 0 is $\text{subgaussian}(\sigma)$ if its moment generating function is not greater than that of a Gaussian with variance $\sigma^2$. A random variable $X$ with nonzero mean is subgaussian if $X - \E[X]$ is subgaussian. 
\end{definition}

Up to constant coefficients in $\sigma$, it can be shown that the following definitions are equivalent for a random variable which is subgaussian with parameter $\sigma$:

\begin{enumerate}
    \item MGF Definition: $M_X(\lambda) \le \exp(\lambda^2\sigma^2/2)$
    \item Tail Definition: $\Pr[|X| > t] \le 2\exp(-t^2/2\sigma^2)$
    \item Moment Definition: $\E[|X|^k] \le k^{k/2}\sigma^k$ for all integers $k\ge 1$
\end{enumerate}

Using this, we can state the two following useful lemmas.

\begin{lemma}
\label{bounded_rv}
Any bounded random variable $X \in [a, a + c]$ is subgaussian with parameter $c/2$. \cite{PR}
\end{lemma}

It directly follows from the above that if $X \in [-1, 1]$, then $\sigma = 1$. 

\begin{lemma}[Addition of Subgaussians]
\label{subgauss_addition}
If we have two independent Subgaussian variables $X$ with parameter $\sigma_1$ and $Y$ with parameter $\sigma_2$, then $X + Y$ is subgaussian with parameter $\sqrt{\sigma_1^2 + \sigma_2^2}$. 
\end{lemma}
\begin{proof}
\begin{align*}
\E[e^{\lambda(X+Y - \E[X + Y])}] &= \E[e^{\lambda (X - \E[X])}]\E[e^{\lambda (Y - \E[Y])}]    \\
&\le e^{\frac{\lambda^2 \sigma_1^2}{2}}e^{\frac{\lambda^2 \sigma_2^2}{2}}\\
&= e^{\frac{\lambda^2(\sigma_1^2+\sigma_2^2)}{2}}\\
&= e^{\frac{\lambda^2(\sqrt{\sigma_1^2+\sigma_2^2})^2}{2}}
\end{align*}
\end{proof}
\begin{corollary} \label{additive_chernoff}
The additive Chernoff bound on independent random variables bounded in $[0, 1]$ holds. 
\end{corollary}
\begin{proof} 
Let $X_1, \ldots, X_n \in [0, 1]$ be independent random variables. Then $X_i$ are subgaussian with parameter $\sigma_i^2 = 1/4$ by Lemma \ref{bounded_rv}. 
Therefore $X = \sum_{i=1}^n X_i$ is subgaussian with parameter $\sigma^2 = n/4$ by Lemma \ref{subgauss_addition}. Therefore
\begin{align*} 
\Pr[|X - \E[X]| \ge t] &\le 2e^{\frac{-t^2}{2(n/4)}}\\
&= 2e^{-\frac{2t^2}{n}}
\end{align*}
by the tail definition of a subgaussian variable. This is exactly the additive Chernoff bound. 
\end{proof}

\bibliographystyle{alpha}

\begin{thebibliography}{42}

\bibitem[1]{MR}
Rajeev Motwani, Prabhakar Raghavan.
\newblock Randomized Algorithms.
\newblock {\em Cambridge University Press}, 0-521-47465-5, 1995.
 
\bibitem[2]{DJ}
David Joyce.
\newblock Moments and the moment generating function.
\newblock {\em Math 217 Probability and Statistics}. Clark University, 2014.

\bibitem[3]{PR}
Philippe Rigollet.
\newblock Lecture Notes.
\newblock {\em High Dimensional Statistics}. MIT, 2015.
 
\end{thebibliography}

\end{document}