\documentclass[11pt]{article}
\usepackage{amsmath,amssymb,amsthm,mathtools}

\DeclareMathOperator*{\E}{\mathrm{E}}
\DeclareMathOperator*{\Var}{\mathrm{Var}}
\DeclareMathOperator*{\argmin}{arg\,min}
\DeclareMathOperator*{\median}{\mathrm{median}}
\DeclareMathOperator*{\mean}{\mathrm{mean}}

\newcommand{\eps}{\epsilon}
\newcommand{\inprod}[1]{\left\langle #1 \right\rangle}
\newcommand{\bbR}{\mathbb{R}}
\newcommand{\poly}{{\rm poly}}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {\bf CS 395T: Sublinear Algorithms } \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[4]{\handout{#1}{#2}{#3}{Scribes: #4}{Lecture #1}}

\newtheorem{thm}{Theorem}[section]
\newtheorem{claim}[thm]{Claim}
\newtheorem{lemma}[thm]{Lemma}
\newtheorem{define}[thm]{Definition}
\newtheorem{cor}[thm]{Corollary}
\newtheorem{obs}[thm]{Observation}
\newtheorem{remark}[thm]{Remark}
\newtheorem{example}[thm]{Example}
\newtheorem{construction}[thm]{Construction}
\newtheorem{conjecture}{Conjecture}
\newtheorem{THM}{Theorem}
\newtheorem{COR}[THM]{Corollary}
\newtheorem{question}{Question}
\newtheorem{fact}[thm]{Fact}
\newtheorem{prop}[thm]{Proposition}
% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex

\newcommand{\codynote}[1]{{\large \{  \textbf{Cody:} #1\}}}

\begin{document}

\lecture{4 --- September 6, 2016}{Fall 2016}{Prof.\ Eric Price}{Cody Freitag, William Swartworth }

\section{Overview}


We previously discussed Markov's Inequality, Chebyshev's Inequality, and Chernoff Bounds, which progressively require more assumptions for tighter concentration at the tails.

\noindent \textbf{Markov's Inequality} 

Let $X \geq 0$ and $t > 0$. Then, $$\Pr[X \geq t] \leq \frac{\E[X]}{t}.$$

\noindent \textbf{Chebyshev's Inequality}

Let $\mu := \E[X]$, $\sigma^2 := \Var[X] = \E[(X-\mu)^2]$, and $t > 0$. Then, $$\Pr[|X - \mu| \geq t] \leq \frac{\sigma^2}{t^2}.$$

\noindent \textbf{Chernoff Bounds}

Let $X = \sum_{i=1}^n X_i$ such that $X_i \in [0, 1]$, fully independent, and $\mu := \E[X]$.  Then, $$\Pr[|X - \mu| > t] \leq e^{-C \frac{t^2}{\sigma^2}}$$ for some constant $C$. 

Today, we will look at how to improve the failure probability for the distinct elements problem from last class.  
Along the way, we will begin to develop some tools that will allow for tighter analysis of the LogLog algorithm.

\section{Improving Distinct Elements}

Recall the LogLog algorithm from last class:

	\begin{enumerate}
		\item For $i = 1$ to $m$, in parallel:
		\begin{enumerate}
			\item Pick a hash function $h_i: [n] \to [0, 1]$.
			\item Record $Y_i = \min_{x \in S} h_i(x)$.
		\end{enumerate}
		\item Compute $\hat{Y} = \frac{1}{m} \sum_{i = 1}^m Y_i$.
		\item Output $1/\hat{Y} - 1$.
	\end{enumerate}
	
Note $n$ is the size of the universe, $S$ is the stream, and $k$ is the true number of distinct elements in the stream.  
We showed last time that $\Var(Y_i) \lesssim 1/k^2$.  Using Chebyshev's Inequality, this implies that $m =  O(1/\epsilon^2)$ trials suffice to get $|\hat{Y} - \mu | \le \epsilon / k$ with probability at least $3/4$.
What if instead we want $|\hat{Y} - \mu | \le \epsilon / k$ with at least $1 - \delta$ probability? We have that $\Var(\hat{Y}) \lesssim \frac{1}{n^2m}$, so Chebyshev's tells us $m = O(\frac{1}{\epsilon^2 \delta})$ trials suffice.

We want to improve the dependency of $m$ on $\delta$ from $1/\delta$ to $\log(1/\delta)$, which we can go about in two different ways.  Either we can find a different algorithm with a better dependence on $\delta$, or we can show that our current algorithm suffices.

\subsection{Median of Means}

We want to find a different algorithm that calculates $\tilde{Y}$ such that $|\tilde{Y} - \mu| \le \epsilon / k$ using fewer samples. We will solve this using the ``median of means'' approach.  So we will split our trials in to $R$ groups, $S_1, \ldots, S_R$, and set $$\tilde{Y} = \median_{i \in [R]} \left(\mean_{j \in S_i} Y_j\right).$$
Let $E_i$ be the indicator of the event $|(\mean_{j \in S_i} Y_j)- \mu| \le \epsilon / k$.  If at least $R/2$ of the $E_i$ events occur, then $|\tilde{Y} - \mu| \le \epsilon / k$.  

Each $E_i \in \{0,1\}$ is independent, and if $|S_i| \ge O(1/\epsilon^2)$, then $\Pr[E_i] \le 1/4$. So using a Chernoff bound, we get that $$\Pr\left [\sum_i E_i \ge R/2\right ] \le e^{-\Omega(R)}.$$ 
This means that if $R \ge O(\log(1/\delta))$, then $|\tilde{Y} - \mu| \le \epsilon / k$ with probability $1-\delta$.  In total, $$m = O\left (\frac{1}{\epsilon^2}\log\frac{1}{\delta}\right)$$ trials suffice.

In fact, we didn't need to take the median of the means in order to use $O\left (\frac{1}{\epsilon^2}\log\frac{1}{\delta}\right)$ trials.  We will now develop some more tools to eventually show why the taking the mean of $O\left (\frac{1}{\epsilon^2}\log\frac{1}{\delta}\right)$ trials works.

\section{Higher Moments}


Suppose $X$ has mean zero, and variance $\sigma^2 = \E[X^2]$.  We can apply Markov's Inequality as we did to prove Chebyshev to get a concentration inequality for $X$.  Namely we have, $$\Pr[|X| \ge t] = \Pr[X^2 \ge t^2] \le \frac{\E[X^2] } {t^2 }.$$

Now suppose that $X = \sum_{i=1}^n X_i$, for $n$ independent, mean-zero $X_i \in [-1,1]$.  Note that $\E[X_i^k] \le 1$ for all values of $k$, and when $k=1$, $\E[X_i] = 0$ since each $X_i$ has mean zero. To apply our second moment bound we would like to estimate $E[X^2]$.  Using only pairwise independence, we have 
$$\E\left[X^2\right] = \E\left[\sum_i X_i^2 + \sum_{i \not = j} X_i X_j\right] = \sum_i \E\left[X_i^2\right] \le n.$$
From our second-moment bound, this implies that $\Pr[|X| \ge t \sqrt{n}] \le 1/t^2.$ 

What if we look at the fourth moment instead?  We can apply Markov's Inequality as before, so
$$\Pr[|X| \ge t] = \Pr[X^4 \ge t^4] \le \frac{\E[X^4]}{t^4}.$$
We can bound $E[X^4]$ now using $4$-wise independence to get
$$\E\left[\left(\sum_i X_i\right)^4\right] = \sum_i \E[X_i^4] + \sum_{i \not = j} 6 \E[X_i^2] \E[X_j^2] \le n + 6n(n-1) \le 6n^2$$
This implies that $\Pr[|X| \ge t \sqrt{n}] \le 6 /t^4$.  Notice that for large $t$ this beats our second moment bound, while for small $t$ the second moment wins.

In fact this method extends to all (even) higher moments.  The same approach with a slightly more careful analysis shows that $$\Pr[|X| \ge t \sqrt{n}] \le \left(\frac{\sqrt{k}}{t}\right)^k$$ for all even $k$.  Furthermore, we only use the fact that the the $X_i$'s are $k$-wise independent. So this is a useful method to concentrate $\sum X_i$ without full independence that the Chernoff Bound requires.  

If our $X_i$ are fully-independent, then for a given value of $t$ we can optimize this bound over $k$.  Setting $k = t^2/e$ implies that $$\Pr[|X| \ge t \sqrt{n}] \le e^{-t^2/(2e)}.$$ 
In fact, Chernoff claims that for $X_i \in [0,1],$ $$\Pr[|y - \mu| \ge \epsilon \mu] \le 2e^{-\epsilon^2/(2+\epsilon) \mu}.$$  Instead, for $X_i \in [-1,1]$ we get $$\Pr[|y - \mu| \ge \epsilon \mu] \le 2e^{-\epsilon^2 \mu^2 /(2en)},$$ which is only worse by a $\mu / n$ factor in the exponent.

\section{Moment Generating Function}

While higher moments can be useful for concentrating random variables, they quickly become cumbersome to work with.  Moment generating functions encapsulate information about all of the moments within a single, much simpler object.

The Moment Generating Function of a mean-zero random variable $X$ is $$\text{MGF}(X)=\E\left [e^{\lambda X}\right] = \E\left[1 + \lambda X + \frac{\lambda^2 X^2}{2!} + \cdots\right].$$

Following the method of the last section, Markov's Inequality tells us that $$\Pr[X > t] = \Pr[e^{\lambda X} > e^{\lambda t}] \le \frac{\E[e^{\lambda X}]}{ e^{\lambda t}}$$ for all $\lambda > 0$.  (And we get a similar lower tail for $\lambda < 0$.)

In the case of a Gaussian $p(t) = \frac{1}{\sqrt{2 \pi \sigma^2}} e^{-t^2/(2\sigma^2)},$ we can express the moment generating function explicitly: 
\begin{align*}
\E[e^{\lambda X}] &= \int_{-\infty}^\infty p(t) e^{\lambda t} dt\\
&= \int_{-\infty}^\infty \frac{1}{\sqrt{2 \pi \sigma^2}} e^{-\frac{1}{2\sigma^2}t^2 + \lambda t} dt\\
&=\int_{-\infty}^\infty \frac{1}{\sqrt{2 \pi \sigma^2}} e^{-\frac{1}{2\sigma^2} (t + \lambda \sigma^2)^2 +\frac{\sigma^2\lambda^2}{2}} dt \;\; \text{ (By completing the square.)}\\
&= e^{\frac{\sigma^2\lambda^2}{2}} \int_{-\infty}^\infty \frac{1}{\sqrt{2 \pi \sigma^2}} e^{-\frac{1}{2\sigma^2} (t + \lambda \sigma^2)^2}dt\\
&= e^{\frac{\sigma^2\lambda^2}{2}} \int_{-\infty}^\infty p(t-\lambda \sigma^2) dt\\
&= e^{\frac{\sigma^2\lambda^2}{2}} \;\; \text{ (Since the integral of a gaussian is $1$)}.
\end{align*}

So we've shown that for $X$ gaussian with mean $0$, $$\text{MGF}(X) = e^{\frac{\lambda^2 \sigma^2}{2}}$$

Therefore, $$\Pr[X \ge t] \le e^{-\frac{t^2}{2\sigma^2}}$$ from setting $\lambda = t/\sigma^2$. 

Combining this with the lower tail, we get the bound $$\Pr[|X| \ge t] \le 2 e^{-\frac{t^2}{2\sigma^2}}.$$

\section{Subgaussian Random Variables}

What if we want to use the MGF to bound a non-gaussian random variable?  It's possible we can compute MGF($X$) exactly, but it could be the case that our random variable is too complicated. We define the notion of a \textit{subgaussian} random variable as one whose moment generating function is bounded by that of a gaussian. 

\begin{define}
A mean-zero random variable $X$ is \textit{subgaussian} with parameter $\sigma^2$ if for all $\lambda$, $$\E[e^{\lambda X}] \le e^{\frac{\lambda^2 \sigma^2}{2}}.$$
\end{define}

 In general, one can show that $X$ is subgaussian (up to constant factors in $\sigma$) if any of the following criteria hold:
 \vspace{ -2 mm }
 \begin{enumerate}
 \item MGF Bound: $\E[e^{\lambda X}] \le e^{\frac{\lambda^2\sigma^2}{2}}$ for all $\lambda$,
 
 \item Tail Bound: $\Pr[|X| \ge t] \le 2 e^{-\frac{t^2}{2 \sigma^2}}$ for all $t > 0$, or
 
 \item Moment Bound: $\E[|X|^k] \le \sigma^k k^{\frac k2}$ for all $k > 0$
 \end{enumerate}

\begin{remark}
A (possibly non-mean-zero) random variable $X$ is \textit{subgaussian} with parameter $\sigma^2$ if $X - \E[X]$ is subgaussian with parameter $\sigma^2$.
\end{remark}
 
 We now give some useful lemmas about subgaussian random variables.
 
 \begin{lemma}
Suppose $X$ is a bounded random variable, i.e. $X \in [a,b]$ for $a,b\in \bbR$.  Then $X$ is subgaussian with parameter $\sigma^2 = \left( \frac{b-a}{2} \right )^2$.
\end{lemma}

 
\begin{lemma}
If $X$ and $Y$ are subgaussian with parameters $s_1^2$ and $s_2^2$, respectively, and independent, then $X+Y$ is subgaussian with parameter $\sigma^2 = s_1^2 + s_2^2$.
\end{lemma}
 
 \begin{proof}
 We'll directly apply the MGF bound to show this.
 \begin{align*}
 \E[e^{\lambda(x+y)}] &= \E[e^{\lambda x} e^{\lambda y}] \\
 	&= \E[e^{\lambda x}] \E[e^{\lambda y}] \\
	&\le e^{\frac{\lambda^2 s_1^2}2} e^{\frac{\lambda^2 s_2^2}2} \\
	&= e^{\frac{\lambda^2(s_1^2 +s_2^2)}2}
\end{align*}
\end{proof}
 
 \subsection{Proof of the Chernoff Bound}
 
 With only the previous two lemmas, we have the machinery in place to prove the following version of the Chernoff Bound.
 
 \begin{thm}
 Let $X = \sum_{i=1}^n X_i$ such that $X_i \in [0,1]$, independent, and $\mu := \E[X]$.  Then, $$\Pr[|X - \mu| > t] \leq e^{-\Omega\left(\frac{t^2}{\sigma^2}\right)}$$
 \end{thm}
 
 \begin{proof}
 Since each $X_i$ is bounded by $[0,1]$, $X_i$ is subgaussian with parameter $\sigma^2 = \left( \frac{1-0}{2} \right)^2 = \frac{1}{4}$.  Since all $X_i$ are independent, $X = \sum_{i=1}^n X_i$ is subgaussian with parameter $\frac{n}{4}$.  This means $$\Pr[|X - \mu|] > t] \le 2e^{-2t^2/n} = e^{-\Omega\left(\frac{t^2}{n}\right)}.$$ 
 \end{proof}
 
Note that by a similar analysis, if some of the $X_i$ are gaussian or on larger ranges, the same theorem holds.

\end{document}
