\documentclass[11pt]{article}
\usepackage{amsmath,amssymb,amsthm}
\usepackage{xcolor}
\usepackage{natbib}
\DeclareMathOperator*{\E}{\mathbb{E}}
\let\Pr\relax
\DeclareMathOperator*{\Pr}{\text{Pr}}
\DeclareMathOperator*{\p}{\text{Pr}}

\newcommand{\eps}{\epsilon}
\newcommand{\inprod}[1]{\left\langle #1 \right\rangle}
\newcommand{\R}{\mathbb{R}}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {\bf CS 388R: Randomized Algorithms, Fall 2019 } \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
      \textcolor{red}{\textbf{NOTE:} THESE NOTES HAVE NOT BEEN EDITED OR CHECKED FOR CORRECTNESS}
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[5]{\handout{XX}{#3}{#4}{Scribe: #5}{Lecture #1: #2}}

\newtheorem{theorem}{Theorem}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{observation}[theorem]{Observation}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{assumption}[theorem]{Assumption}
\usepackage[utf8]{inputenc}

\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{amsthm}
\usepackage{wasysym}
\usepackage{geometry}
\usepackage{color}
\usepackage{hyperref}
\usepackage{bm}
\usepackage{algorithm}
\usepackage{algorithmic}
\usepackage{enumitem}
\hypersetup{
    colorlinks=true,
    linkcolor=blue,
    filecolor=magenta,      
    urlcolor=cyan,
}

\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex

\newcommand*{\dif}{\mathop{}\!\mathrm{d}}
% \geometry{a4paper,scale=0.8}
\title{Randomized Algorithms Lecture 2}
\author{Will Sherwood, Feifei Zhang}
\date{September 1, 2019}

\begin{document}

\lecture{2}{Concentration inequalities}{September 1, 2019}{Prof.\ Eric Price}{Will Sherwood, Feifei Zhang}
% \maketitle

\section{Concentration inequalities}

\subsection{Markov's Inequality}

Let $X$ be a nonnegative random variable. Then, for any $t > 0$,

$$\p[X \ge t] \le \frac 1 t \E[x].$$

\begin{proof}
$$\E[X] =   \int_0^\infty f_X(s)\cdot s\ \text{d}s
        \ge \int_t^\infty f_X(s)\cdot s\ \text{d}s 
        \geq \int_t^\infty f_X(s)\cdot t\ \text{d}s=\p[X \ge t].$$
\end{proof}

\subsection{Chebyshev's Inequality}

Let $X$ be a random variable with variance $\sigma^2$ and finite expectation. Then, for any $t > 0$,

$$\text{Pr}[|X-\E[X]| > t\sigma] \le \frac 1 {t^2}.$$
\begin{proof}
Let $Y$ denote the non-negative random variable $|X - \E[X]|$. By Markov's Inequality, 
$$\p[|X-\E[X]|\geq t\sigma]=\p[Y \ge t\sigma] = \p[Y^2 \ge t^2 \sigma^2] \le \frac{1}{t^2\sigma^2} \E[Y^2] = \frac 1 {t^2}.$$
\end{proof}

\subsection{Chernoff Bounds}

Let $X$ be a random variable with expectation $\mu$. Then,


$$\p[X - \mu \ge t] \le \min_{\lambda > 0} \frac{\E[e^{\lambda(X-\mu)}]}{e^{\lambda t}}.$$

\begin{proof}
$$\p[X - \mu \ge t] = \p[e^{X - \mu} \ge e^t] \le \frac{\E[e^{\lambda(X-\mu)}]}{e^{\lambda t}} \text { for } \lambda > 0 \text{ by Markov's inequality.}$$
\end{proof}

\textbf{Example}: Let $x_i \sim [0,1]$ be independently and identically distributed random variables and $x = \sum_{i \in [n]} x_i$. Then, for any $t \ge 0$,

$$\p[x \ge E[x] + t] \le e^{-2t^2/n}$$
$$\p[x \le E[x] - t] \le e^{-2t^2/n}$$

These are the \emph{additive} Chernoff bounds.

\begin{align*}
  \p[x \ge (1+t)E[x]] &\le e^{-\E[x] t^2 / (2 + t)}\\
  \p[x \le (1-t)E[x]] &\le e^{-\E[x] t^2 / 2}
\end{align*}

These are the \emph{multiplicative} Chernoff bounds, which often achieve a better lower bound on probability when $E[x]$ is small.

\section{Examples}
\subsection{Unbiased coin}
Flip a fair coin 1000 times. Let $X$ be the number of heads.
\begin{itemize}
    \item $\E[X]=1000 \cdot 0.5 = 500$.
    \item $\p[X = 500] = { 1000 \choose 500} / 2^{1000} \approx 0.025$.
    
    \item Question: Pr[X $\geq$ 600] = ? \\\\
    We apply additive Chernoff bounds $(n = 1000, \mu = 500)$
    
    \begin{align*}
        \p[X \geq 600]
          & \leq  e^{\frac{-2 \cdot 100^{2}}{1000}} \\
          & =  e^{-20}
    \end{align*}
    
    Note that we can get the same bound with the central limit theorem.
    $$ X \sim \mathcal{N}(\mu = 500, \sigma^{2} = 250) $$ 
    \begin{align*}
        \p[X \geq 600]
          & \leq e^{\frac{\left(\frac{100}{\sqrt{250}}\right)^{2}}{2}} \\
          & =  e^{-20}
    \end{align*}
    
    Applying the multiplicative Chernoff bound gives a less precise upper bound.
    \begin{align*}
        \p[X \geq 600]
          & =  Pr[X \geq (1 + 0.2) \cdot 500] \\
          & \leq e^{-\frac{0.2^{2}}{2 + 0.2} \cdot 500} \\
          & =  e^{-9.091}
    \end{align*}
    
\end{itemize}

\subsection{Biased coin}

Suppose $n$ $p$-biased coins are flipped. Let $X$ denote the number of heads shown, and $\mu := \E[X]$.
In this example, we directly compare the bounds received from both additive and multiplicative Chernoff:
For any $k \ge 0$,

\begin{enumerate}[leftmargin=\parindent,align=left,labelwidth=\parindent,labelsep=0pt]
    \item[(Multiplicative Chernoff).]  
    \begin{center}\hspace{0.9in}
        $\p[X \ge \mu + k] \le e^{-k^2/(2\mu + k)}.$

    \item[(Additive Chernoff).] \hspace{1in} $\p[X \ge \mu + k] \le e^{-2k^2/n}.$
    \end{center}
\end{enumerate}


In particular, using the multiplicative bound we have $$\p[X = n] = p^n \le e^{-\frac{(p-1)^2}{p+1}n},$$ and using the additive bound gives  $$\p[X = n] = p^n \le e^{-2(p-1)^2n}$$


\begin{proof}
\begin{flalign*}
  && \p[X \ge \mu + k] & = \p[X \ge ( k /\mu + 1)\mu] && \\
  && & \le e^{\frac{-\mu(k /\mu)^2}{2 + k/\mu}} && (\text{Multiplicative Chernoff bound}) \\
  && & = e^{\frac{-k^2}{2\mu+k}}. && \\
\end{flalign*}
The tail bounds above comes from the application $k = n - \mu$ and $\mu=np$.
\end{proof}

\subsection{Amplification with confidence $1-\delta$}
Suppose we have an algorithm $A$ which solves some optimization problem correctly with probability
$\frac{2}{3}$ with no guarantees otherwise. For any $\delta > 0$, the following algorithm solves
the optimization problem correctly with probability $1-\delta$:

\textbf{Algorithm}: Given an input to the optimization problem $x$, execute $A(x)$ at least $N=18 \cdot \log(1/\delta)$ times and return the majority output. If there is no majority, return nothing.

\begin{proof}

Suppose we run $A$ $n$ times and take the majority and let $X$ be the number of successful times. The probability that the algorithm fails would be
\begin{flalign*}
 && 
   &\p\left[\text{Amplification algorithm fails}\right] && \\
 && \leq& \p\left[X \leq \frac{n}{2}\right] && \\
 && = &\p\left [X \leq E\left[X\right] - \frac{n}{6}\right] && (E\left[X\right] = 2n/3) \\
 && \leq& e^{-\frac{-2\cdot(n/6)^{2}}{n}} =  e^{-\frac{n}{18}} && (\text{Chernoff Bound}) \\
\end{flalign*}
Hence, setting
$$ n \geq 18 \cdot \log(1/\delta)$$
lets the majority algorithm succeed with probability $1-\delta$.
\end{proof}

\subsection{Biased coin: estimate of $p$}
Suppose biased coin with unknown $ p \in [0, 1] $. \\
How many trials ($n$ = ???) are needed to estimate $p$ to some error $ \pm \epsilon $, with probability $1-\delta$? \\
Given $n$ i.i.d. samples, with $\p[x_{i}] = p $
$$ x_{1}, ... x_{n} \in \{0, 1\} $$

For some set error $\epsilon$ and probability $1 - \delta$, for what $n$ it is thre that $$\p\left[\left|\frac{1}{n} \sum x_i - p \right| < \epsilon \right]\geq 1-\delta?$$

We can provide an upper bound with additive Chernoff bounds.

\begin{align*}
 \p\left[\left|\frac{1}{n}\sum x_i - p\right| \geq \epsilon \right]
  &  = \p\left[\left|\sum x_i - p \right| \geq \epsilon \cdot n\right] \\
  & \leq 2 \cdot e^{-2 \cdot \frac{(\epsilon \cdot n)^{2}}{n}} \\
  & = 2 \cdot e^{-2 \epsilon^{2} \cdot n} 
\end{align*}


Note that the constant factor of 2 comes from adding the two tails of this distribution 
$$ \p\left[\frac{1}{n} \sum X_i - p > \epsilon\right] + \p\left[\frac{1}{n} \sum X_i - p < -\epsilon\right]$$

Now we solve for $n$. To make
$$\p\left[\left|\frac{1}{n}\sum x_i - p \right| \geq \epsilon \right] 
    \leq 2 \cdot e^{-2 \epsilon^{2} \cdot n} \leq \delta,$$
it is enough to set
$$ n \geq \frac{1}{2\cdot\epsilon^{2}} \log\frac{2}{\delta} $$

We can also provide an estimate using multiplicative Chernoff bounds.

$$ \p\left[\left|\frac{1}{n}\sum x_i - p\right| \geq \epsilon \cdot p\right]
    \leq 2 \cdot e^{-\frac{\epsilon^{2}}{3} \cdot p \cdot n}
     \text{, for $ \epsilon <$  1} $$
Substituting $\epsilon p$ by $p$, we have
$$ \p\left[\left|\frac{1}{n}\sum x_i - p\right| \geq \epsilon\right]
    \leq 2 \cdot e^{-\frac{\epsilon^{2}}{3} \cdot \frac{n}{p}} \text{, for $ \epsilon <p$} $$
To let the probability bounded by $\epsilon$, set
$$ n \geq \frac{p}{\epsilon^{2}} \log(\frac{2}{\delta})$$

Note that this is similar to the additive Chernoff bound. However, the multiplicative bound may provide a better upper bound in the case of small enough $p$.

\end{document}