\documentclass[11pt]{article}
\usepackage{amsmath,amssymb,amsthm}

\DeclareMathOperator*{\E}{\mathbb{E}}
\DeclareMathOperator*{\var}{Var}
\let\Pr\relax
\DeclareMathOperator*{\Pr}{\mathbb{P}}

\newcommand{\eps}{\epsilon}
\newcommand{\abs}[1]{|#1|}
\newcommand{\norm}[1]{\|#1\|}
\newcommand{\inprod}[1]{\left\langle #1 \right\rangle}
\newcommand{\R}{\mathbb{R}}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {\bf CS 395T: Sublinear Algorithms } \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[4]{\handout{#1}{#2}{#3}{Scribe: #4}{Lecture #1}}

\usepackage{framed}
\newtheorem{theorem}{Theorem}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{observation}[theorem]{Observation}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{definition}[theorem]{Definition}
\newenvironment{example}[1][\unskip]
{\begin{framed}\textbf{Example: #1.}}
{\end{framed}}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{assumption}[theorem]{Assumption}

% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex

\begin{document}

\lecture{2 --- September 2, 2014}{Fall 2014}{Prof.\ Eric Price}{Ben Leedom}

\section{Markov and Chebyshev Inequalities}
Let $X \geq 0$ be a nonnegative random variable. For all $t \geq 0$,
$\E[X] \geq t \cdot \Pr[X \geq t]$, so

\begin{align}
  \label{eq:1}
  \Pr[X \geq t] \leq \frac{\E[x]}{t}.
\end{align}
Although it is trivial, this bound appears all over the place.  Hence
it gets a name: \emph{Markov's inequality}.

Let $\mu := \E[X], \var(X) := \E[(x - \mu)^2]$.  We have that
\begin{align}
  \label{eq:chebyshev}
  \Pr[|x - \mu| \geq t] = \Pr[(x - \mu)^2 \geq t^{2}] \leq
  \frac{\var(X)}{t^2}
\end{align}
which is known as \emph{Chebyshev's inequality}.  This will appear
very often throughout the course.

In general, one can take arbitrary moments:
\begin{align}
  \Pr[|x - \mu| \geq t] = \Pr[|x - \mu|^{k} \geq t^{k}] \leq
  \frac{\E[|x - \mu|^{k}]}{t^{k}}\label{eq:higher}
\end{align}
and doing so for $k \geq 3$ is known as a \emph{higher moment method}.

As an example for how these moment methods work, consider $X \sim N(0,
1)$ (with PDF $p(t) = \frac{1}{\sqrt{2\pi}}e^{-\frac{t^2}{2}}$). Then
$\E[x^{k}] \approx k^{\frac{k}{2}}$, so the $k$th moment method gives:
\begin{align}\label{highermoment}
  \Pr[x \ge t] ~\lesssim (\frac{k}{t^{2}})^{\frac{k}{2}}
\end{align}
Therefore the best moment $k$ in this case depends on $t$: choosing $k
= t^2/2$ gives a bound of $\Pr[x \geq t] \lesssim 2^{-t^2/4}$, while
substantially larger or substantially lower values of $k$ give worse
bounds.

In most cases, a simpler method than optimizing the choice of $k$ in a
higher moment method is to use a \emph{moment generating function}.

\section{Moment Generating Functions}

Define the moment generating function $\Phi(\lambda)$ like this:
$$
\Phi(\lambda) = \E[e^{\lambda(x - \mu)}]
$$

By Markov's inequality,
\begin{align}
  \label{eq:mgf}
\Pr[x - \mu \geq t] = \Pr[e^{\lambda(x - \mu)} \geq e^{\lambda t}] \leq \frac{\Phi(\lambda)}{e^{\lambda t}}
\end{align}
where we can optimize over $\lambda$.


Note that $e^{\lambda x} = 1 + \lambda x + \frac{\lambda^2 x^2}{2} +
\frac{\lambda^3 x^3}{3!} + \dotsb {}$ is a weighted average of the
moments of $x$.  Therefore optimizing $\lambda$ in~\eqref{eq:mgf} is
analogous to optimizing $k$ in~\eqref{highermoment}.  It's actually
slightly weaker: the best higher moment bound is at least as good as
the best moment generating function bound.  However, we shall see that
the moment generating bound is much more convenient, and usually does
a good enough job.

\begin{example}[Gaussian Variables]
Consider the normal distribution $N(0, \sigma^2)$, given by the PDF
$p(t) = \frac{1}{\sqrt{2 \pi \sigma^2}} e^{-\frac{t^2}{2 \sigma^2}}$.
The moment generating function is
\begin{align}
\Phi(\lambda) &= \int_{-\infty}^{-\infty} \frac{1}{\sqrt{2 \pi \sigma^2}} e^{-\frac{t^2}{2 \sigma^2}}e^{\lambda t} dt
= \int_{-\infty}^{\infty} \frac{1}{\sqrt{2 \pi \sigma^2}} e^{-[\frac{(t - \sigma^2\lambda)^2}{2 \sigma^2} + \frac{\sigma^2 \lambda^2}{2}]} dt
= e^{\frac{\sigma^2 \lambda^2}{2}}
\end{align}

Thus
\[
\Pr[x \ge t] \le \inf_{\lambda} \frac{\Phi(\lambda)}{e^{\lambda t}} = \inf_{\lambda} e^{\frac{\lambda^2 \sigma^2}{2} - \lambda t} = \inf_{\lambda} e^{\frac{1}{2}(\lambda \sigma - \frac{t}{\sigma})^2} \cdot e^{-\frac{t^2}{2 \sigma^2}}
\]
But of course
$\inf_{\lambda} e^{\frac{1}{2}(\lambda \sigma - \frac{t}{\sigma})^2} = 1$, so this gives us
\begin{align}
  \label{eq:subg}
  \Pr[x \ge t] \le e^{-\frac{t^2}{2 \sigma^2}}
\end{align}
with the choice of $\lambda = \frac{t}{\sigma^2}$.
\end{example}

\section{Subgaussian variables}
We say a random variable is ``subgaussian'' if its moment generating
function is bounded by that of a gaussian:

\begin{definition}
  A variable $X$ is \emph{subgaussian with parameter $\sigma$} if for
  all $\lambda$,
  \[
  \Phi(\lambda) \le e^{\frac{\lambda^2 \sigma^2}{2}}.
  \]
\end{definition}
As with~\eqref{eq:subg}, for subgaussian $X$ we have
\[
\Pr[x \ge \mu + t] \le e^{-\frac{t^2}{2 \sigma^2}}
\]
and (by replacing $X$ with $-X$) we also have
\[
\Pr[x \le \mu - t] \le e^{-\frac{t^2}{2 \sigma^2}}.
\]

\begin{example}[bounded random variables]  If $x \in \{-1, 1\}$
uniformly at random, then $x$ is subgaussian with parameter $\sigma =
1$.  More generally, if $x \in [a, b]$, then $x$ is subgaussian with
parameter $\sigma = \frac{b-a}{2}$.  The proof is left as an exercise.
\end{example}


If $X_1$ and $X_2$ are independent and subgaussian with parameters
$\sigma_1$ and $\sigma_2$, then $X_1 + X_2$ is subgaussian with
parameter $\sigma = \sqrt{\sigma_1^2 + \sigma_2^2}$. Proof:
\[
\E[e^{\lambda(X_1 + X_2)}] = \E[e^{\lambda X_1}e^{\lambda X_2}] = \E[e^{\lambda X_1}]\E[e^{\lambda X_2}] \le e^{\lambda^2 \sigma_1^2}e^{\lambda^2 \sigma_2^2} = e^{\lambda^2(\sigma_1^2 + \sigma_2^2)}
\]
This property is why the moment generating function is so nice to work
with.



\begin{example}[coin flips] suppose we flip $n$ coins, getting $x_1,
  \dotsc, x_n \in \{0, 1\}$. We expect the number of heads $\sum x_i$ to be about $n/2$, but how well does it concentrate about the mean?   What is a bound on
  \[
  \Pr[\Sigma \ge (\frac{1}{2} +
  \epsilon)n]?
  \]

  Well, each $x_i$ is bounded, so it is subgaussian with parameter
  $\sigma = \frac{1}{2}$. Since the $x_i$ are independent, the sum has
  parameter $\sigma = \frac{\sqrt{n}}{2}$.  Therefore
  $$
  \Pr[\sum x_i \ge \frac{n}{2} + \epsilon n] \le e^{\frac{-(\epsilon n)^2}{2(\frac{\sqrt{n}}{2})}} = e^{-2\epsilon^2 n}
  $$

How many samples do we need to determine $\mu$ to $\pm \epsilon$ with $(1 - \delta)$ probability?

$$
n = \frac{1}{2\epsilon^2} \log \frac{2}{\delta} = \Theta(\frac{1}{\epsilon^2} \log \frac{1}{\delta})
$$
\end{example}

The logic in this example is general, giving the \emph{hoeffding bound}:
\begin{theorem}[Hoeffding]
Let $X_i$ for $i = 1,\dotsc,n$ be independent subgaussian variables
with mean $\mu_i$ and parameter $\sigma_i$ (for example, be bounded in
the range $[a_i - \sigma_i, a_i + \sigma_i]$). Then
\[
\Pr[\sum_i{(x_i - \mu_i)} > t] \le e^{-\frac{t^2}{2\sum_{i} \sigma_i^2}}
\]
\end{theorem}


\section{Alternate definitions of subgaussian variables}
One might try to define ``subgaussian'' variables to be ones that are
bounded by a Gaussian in other ways than the moment generating
function.  For instance, you might consider bounding the tail
probability or the individual moments:

(Bounded Tail): $X$ is subgaussian if for some constant $C$ and
parameter $\sigma$, with $Z \sim N(0, \sigma^2)$ and for all $t$,
\[
\Pr[|X| \ge t] \le C \Pr[|Z| \ge t].
\]

(Moments): $X$ is subgaussian if for some parameter $\theta$, for all
$k$ we have
\[
\E[X^{2k}] \le \frac{(2k)!}{(2^k k!)} \theta^{2k}.
\]

Fortunately, these definitions are \emph{equivalent} (up to
constants).  So it suffices to prove that any of these properties
hold.

\section{Sub-exponential variables}
Some variables do not concentrate as well as a gaussian.  A useful
subset of these are variables that concentrate as well as
\emph{exponential} random variables.

\begin{definition}
  A variable $X$ is \emph{sub-exponential with parameters $\sigma$,
    $B$} if $\E[e^{\lambda(x - \mu)}] \le e^{\frac{\sigma^2
      \lambda^2}{2}}$ for all $|\lambda| \le B$.
\end{definition}

\begin{example}[exponential random variable]
Let $X$ be a continuous variable with $p(x) = ce^{-cx}$, for $x \ge 0$, so that $\E[x] = \frac{1}{c}$. Then
$$
\E[e^{\lambda x}] = c \int_0^{\infty} e^{\lambda x}e^{-cx} dx = \frac{c}{c - \lambda}
$$
when $\lambda < c$, and so
$$
\E[e^{\lambda(x - \mu)}] = \frac{ce^{-\frac{\lambda}{c}}}{c - \lambda}
$$
Plotting it, we see that for all $|\lambda| \le \frac{1}{2}$.
\[
\E[e^{\lambda(x - \mu)}] \le e^{\lambda^2}.
\]
Therefore $X$ is subexponential with parameters $(\sqrt{2}, 1/2)$.
\end{example}

\begin{example}[square of a gaussian]
Example: let $Z \sim N(0, 1)$ and $X = Z^2$.
\begin{align}
\E[e^{\lambda(x - 1)}] &= \frac{1}{\sqrt{2 \pi}} \int_{-\infty}^{-\infty} e^{\lambda(z^2 - 1)}e^{-\frac{z^2}{2}} dz
= \frac{e^{-\lambda}}{\sqrt{1 - 2\lambda}}
\end{align}
Plotting the function, we can observe that for all $|\lambda| \le \frac{1}{4}$,
\[
\E[e^{\lambda(x - 1)}]  \le e^{\frac{4\lambda^2}{2}}.
\]
Therefore $X$ is subexponential with parameters $(2, 1/4)$.
\end{example}

Using the same idea as the proof of~\eqref{eq:subg}, but requiring
$\abs{\lambda} \in [-B, B]$, we get a tail bound that contains two
terms: the same as for subgaussian random variables if the optimal
$\lambda = t/\sigma^2 \in [-B, B]$, and a simple exponential
otherwise.  In particular, we have
\[
\Pr[x \ge \mu + t] \le \left\{
  \begin{array}{cl}
    e^{-\frac{t^2}{2\sigma^2}} &\text{if } 0 \leq t \le B\sigma^2\\
    e^{-\frac{Bt}{2}} &\text{if }t \ge B\sigma^2
  \end{array}
  \right.
\]
and a similar bound for the lower tail, after negating $t$.

Just as for subgaussians, if $X_1, X_2$ are sub-exponential with
parameters $(\sigma_1, B_1), (\sigma_2, B_2)$, then $X_1 + X_2$ is
sub-exponential with parameters $(\sqrt{\sigma_1^2 + \sigma_2^2},
\min(B_1, B_2))$.

\section{Johnson-Lindenstrauss Theorem}

\begin{lemma}[Johnson-Lindenstrauss '84]
  Let $x_1, \dotsc, x_n \in \R^d$.  There exist $y_1, \dots, y_n \in
  \R^m$ such that
  \[
  \norm{y_i - y_j}_2 = (1 \pm \epsilon)\norm{x_i - x_j}_2 \forall i,j
  \]
  with $m = O(\frac{1}{\epsilon^2} \log n)$.  Moreover, such $y_i$ can
  be computed efficiently.
\end{lemma}

\begin{proof}
  Let $A \in \R^{m \times d}$, $A_{ij} \sim N(0, \frac{1}{m})$.  We
  simply set $y_i = Ax_i$.

  To see this works, consider any $z \in \R^d$.  We have for each
  coordinate $i$ that
  \[
  (Az)_i = \sum_j A_{ij}z_i \sim N(0, \frac{\norm{z}_2^2}{m}),
  \]
  and the coordinates are independent.

  Let $w = N(0, I_m) = (Az) \cdot \frac{\sqrt{m}}{\norm{z}_2}$.  We
  would like to show that $\norm{w}_2^2$ concentrates about its mean.

  Recall from the example that for each $i$, $w^2_i$ is the square of
  a standard normal, and hence subexponential with parameters $(2,
  1/4)$.  Therefore $\norm{w}_2^2 = \sum w_i^2$ is sub-exponential
  with parameters $(2\sqrt{m}, \frac{1}{4})$. Then
  \[
  \Pr[\norm{w}_2^2 \ge m + \epsilon m] \le e^{\frac{-\epsilon^2 m^2}{2
      \cdot 4m}} = e^{\frac{-\epsilon^2 m}{8}}
  \]
  So $\norm{Az}_2^2 = (1 \pm \epsilon)\norm{z}_2^2$ with probability
  $1 - 2e^{\frac{-\epsilon^2 m}{8}}$.  This is true for any $z$,
  including $z = x_i - x_j$ for each $i, j$.  Setting $m =
  \frac{8}{\epsilon^2} \log \frac {n^3}{2} =
  \Theta(\frac{1}{\eps^2}\log n)$ gives probability $1 - \frac{1}{n}$
  that for all $i,j$
  \[
  \norm{y_i - y_j}_2^2 = (1 \pm \epsilon)\norm{x_i - x_j}_2^2
  \] 
\end{proof}

\end{document}
