\documentclass[letterpaper,11pt]{article}
\input{latexdefs}

\begin{document}
\makeheader{Basic Probability Fact Sheet}

\subsection*{Basic Definitions}
\begin{itemize}
  \item A {\em finite probability space} $(\Omega, p)$ consists of a finite set
  $\Omega = \set{\omega_1, \ldots, \omega_n}$ and a {\em probability mass function}
  $p \colon \Omega \to [0, 1]$ such that $\sum_{\omega \in \Omega} p(\omega) = 1$.
  We refer to $\Omega$ as the {\em sample space} and $\omega_i$ as a possible
  {\em outcome} of a probabilistic event. Throughout this handout, we will only
  consider finite probability spaces.

  \item An {\em event} $E$ over a probability space $(\Omega, p)$ is a set $A \subseteq \Omega$.
  The probability of event $E$, denoted $\Pr[E]$ is defined to be
  $\Pr[E] \deq \sum_{\omega \in E} p(\omega)$. For an outcome $\omega \in \Omega$,
  we will write $\Pr[\omega]$ to denote $p(\omega)$.

  \item A {\em random variable} $X$ over a probability space $(\Omega, p)$ is a
  real-valued function $X \colon \Omega \to \R$. For the remainder of this handout, we will
  assume all random variables are defined over a probability space $(\Omega, p)$.
\end{itemize}

\subsection*{Expected Value and Variance}
\begin{itemize}
  \item The {\em expected value} $\E[X]$ of a random variable $X$ is defined to be
  \[ \E[X] \deq \sum_{\omega \in \Omega} X(\omega) \Pr[\omega]. \]

  \item {\bf Linearity of expectation:} For all random variables $X, Y$
  and all $\alpha, \beta \in \R$,
  \[ \E[\alpha X + \beta Y] = \alpha \, \E[X] + \beta \, \E[Y]. \]

  \item The {\em variance} $\Var(X)$ of a random variable $X$ is defined to be
  \[ \Var(X) \deq \E \left[(X - \E[X])^2 \right] = \E[X^2] - E[X]^2 \]
\end{itemize}

\subsection*{Useful Bounds}
\begin{itemize}
  \item {\bf Union bound:} For every collection of events $E_1, \ldots, E_n$,
  \[ \Pr \Bigg[\bigcup_{i \in [n]} E_i \Bigg] \le \sum_{i \in [n]} \Pr[E_i]. \]

  \item {\bf Markov's inequality:} Let $X$ be a non-negative random variable.
  For all $t > 0$, 
  \[ \Pr[X \ge t] \le \frac{\E[X]}{t}. \]

  \item {\bf Chebyshev's inequality:} Let $X$ be a random variable. For all $t > 0$,
  \[ \Pr[\abs{X - \E[X]} \ge t] \le \frac{\Var(X)}{t^2}. \]

  \item {\bf Chernoff bounds:} Let $X_1, \ldots, X_n$ be independent binary-valued random variables
  (i.e., the value of $X_i$ is either $0$ or $1$). Let $X = \sum_{i \in [n]} X_i$
  and $\mu = \E[X]$.
  Then, for every $t > 0$,
  \[\Pr[X \ge (1 + t) \, \mu] \le \left[ \frac{e^t}{(1 + t)^{1 + t}} \right]^\mu \qquad \qquad
    \Pr[X \le (1 - t) \, \mu] \le \left[ \frac{e^{-t}}{(1 - t)^{1 - t}} \right]^\mu. \]
  Often, the following simpler (and looser) bounds suffice:
  \begin{align*}
    \forall 0 \le t \le 1, &\quad \Pr[X \le (1 - t) \mu] \le e^{-\frac{t^2 \mu}{2}} \\
    \forall 0 \le t, &\quad \Pr[X \ge (1 + t) \mu] \le e^{-\frac{t^2 \mu}{2 + t}}.
  \end{align*}
  Another useful variant (by Hoeffding) gives a bound on the sum
  of any sequence of bounded random variables. Specifically, let $X_1, \ldots, X_n$
  be independent random variables where each $X_i \in [a_i, b_i]$ for $a_i, b_i \in \R$.
  As before let $X = \sum_{i \in [n]} X_i$ and let $\mu = \E[X]$. Then,
  for all $t > 0$,
  \[ \Pr\left[ \abs{X - \mu} \ge t \right] \le 2 \exp\left( -\frac{2t^2}{\sum_{i \in [n]} (b_i - a_i)^2} \right). \]
  For the special case where $X_i \in [0, 1]$ for all $i \in [n]$, the bound becomes
  \[ \Pr\left[ \abs{X - \mu} \ge t \right] \le 2 e^{-2t^2/n}. \]
\end{itemize}

\begin{example}
  Suppose $X_1, \ldots, X_N$ are independent binary-valued
  random variables where $\Pr[X_i = 1] = \frac{1}{2} + \varepsilon$.
  Let $\bar X = \frac{1}{N} \sum_{i \in [N]} X_i$. If $N = \lambda / \varepsilon^2$, then
  \[ \Pr[\bar X \ge 1/2 + \varepsilon/2] \ge 1 - \negl(\lambda). \] This follows by a direct
  application of the Chernoff/Hoeffding bound:
  \[ \Pr \left[ \bar X < \frac{1}{2} + \frac{\varepsilon}{2} \right] =
     \Pr \left[ \sum_{i \in [N]} X_i - N \left(\frac{1}{2} + \varepsilon \right) < -\frac{\varepsilon}{2}N \right] \le
     2 e^{-\varepsilon^2 N^2 / 2N} = 2 e^{-\lambda / 2} = \negl(\lambda). \]
\end{example}

\subsection*{Averaging Argument}

The basic averaging argument states that
if $X_1, \ldots, X_n \in \R$ are values with mean $\mu = \frac{1}{n} \sum_{i \in [n]} X_i$,
then there exists at least one $i \in [n]$ where $X_i \ge \mu$. There are several
variants of this fact that often come in handy:

\begin{lemma}
  \label{lem:averaging-argument}
  Let $X_1, \ldots, X_n \in [0, 1]$ whose average is $\mu$. Then at least an $\varepsilon$-fraction
  of the $X_i$'s are at least $p$ where $\varepsilon = \frac{\mu - p}{1 - p}$.
\end{lemma}
\begin{proof}
  Let $t$ be the fraction of $X_i$'s where $X_i \ge p$. Then,
  $\mu < (1 - t) p + t = p + (1 - p) t$, so $t > (\mu - p) / (1 - p)$.
\end{proof}

\noindent We state two immediate corollaries of \cref{lem:averaging-argument}
that are often useful:
\begin{corollary}
  If $X_1, \ldots, X_n \in [0, 1]$ whose average is $\mu$,
  then at least a $(\mu / 2)$-fraction of the $X_i$'s are at least $\mu / 2$.
\end{corollary}

\begin{corollary}
  \label{lem:averaging-argument-2}
  Let $X_1, \ldots, X_n \in [0, 1]$ whose average is $\mu = p + \varepsilon$.
  Then, at least an
  $\frac{\varepsilon}{2 (1 - p - \varepsilon / 2)} > \frac{\varepsilon}{2 (1 - p)}$ fraction
  of the $X_i$'s are at least $p + \varepsilon / 2$.
\end{corollary}

\begin{example}
  Let $f$ be a function. Suppose we have an algorithm $\calA$ where
  \[ \Pr[ x \getsr \zo^n, y \getsr \zo^n : \calA(x, y) = f(x)] = \frac{11}{12}. \]
  We say a string $y^* \in \zo^n$ is ``good'' if
  \[ \Pr[x \getsr \zo^n : \calA(x, y^*) = f(x)] \ge \frac{3}{4}. \]
  By an averaging argument (\cref{lem:averaging-argument}), at least
  a $2/3$-fraction of $y$'s are good (i.e., set $\mu = 11/12$ and $p = 3/4$).
  Namely,
  \[ \Pr\big[ y \getsr \zo^n : \Pr[x \getsr \zo^n : \calA(x, y) = f(x)] \ge 3/4 \big] \ge 2/3. \]
\end{example}

\begin{example}
  Let $f$ be a function. Suppose we have an algorithm $\calA$ where
  \[ \Pr[x \getsr \zo^n, y \getsr \zo^n : \calA(x, y) = f(x)] = \frac{1}{2} + \varepsilon. \]
  We say that a string $y^* \in \zo^n$ is ``good'' if
  \begin{equation*}
    \Pr[x \getsr \zo^n : \calA(x, y^*) = f(x)] \ge \frac{1}{2} + \frac{\varepsilon}{2}.
  \end{equation*}
  By an averaging argument (\cref{lem:averaging-argument-2}), at least
  an $\varepsilon$-fraction of $y$'s are good. Namely,
  \[ \Pr\big[ y \getsr \zo^n :
     \Pr[ x \getsr \zo^n : \calA(x, y) = f(x)] \ge 1/2 + \varepsilon / 2 \big] \ge \varepsilon. \]
\end{example}

\end{document}
