\documentclass[11pt]{article}
\usepackage{amsmath,amssymb,amsthm}

\DeclareMathOperator*{\E}{\mathbb{E}}
\let\Pr\relax
\DeclareMathOperator*{\Pr}{\mathbb{P}}
\DeclareMathOperator*{\var}{\mathrm{Var}}

\newcommand{\eps}{\epsilon}
\newcommand{\inprod}[1]{\left\langle #1 \right\rangle}
\newcommand{\R}{\mathbb{R}}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {\bf CS388R: Randomized Algorithms } \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[4]{\handout{#1}{#2}{#3}{Scribe: #4}{Lecture #1}}

\newtheorem{theorem}{Theorem}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{observation}[theorem]{Observation}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{assumption}[theorem]{Assumption}

% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex

\begin{document}

\lecture{5 --- Sep 14, 2015}{Fall 2015}{Prof.\ Eric Price}{Chao-Yuan Wu, Jin Zhang}

\section{Overview}

In the last lecture we introduced coupon collector problem, concentration in equalities, subguassian random variables and their basic properties.

In this lecture we will see these properties are equivalent up to a scaling factor and introduce subexponential and subgamma random variables.

\section{Starter Problem}
\subsection{}
\paragraph{Question: }
Suppose $x$ is drawn from an unknown distribution with mean $\mu$ and variance $\sigma^2$. How many samples do we need to estimate $\mu \pm \eps \sigma$ with probability $\geq 1 - \delta$?

\paragraph{One solution: }
Let $\bar{x} := \frac{\sum x_i}{n}$. Then
\begin{align}
\var[\bar{x}] &= \frac{\sigma^2}{n}\\
\E[\bar{x}] &= \mu
\end{align}

By Chebyshev inequality, 
\begin{align*}
\Pr(|\bar{x} - \mu| \ge \alpha \frac{\sigma}{\sqrt{n}}) \le \frac{1}{\alpha^2}. 
\end{align*}
Let $\delta = \frac{1}{\alpha^2}$, and we conclude that
\begin{align*}
n \ge \frac{1}{\delta \eps^2}
\end{align*}

\subsection{Is this bound tight?}
Let's look at an example.
Suppose $x \in \{0, 1\}$ with $\Pr(x = 1) = p = \frac{1}{10n^2}$. Then we have
\[\sigma = \sqrt{p(1-p)} \approx \sqrt{p} < \frac{1}{3n}\]

If any of the sample is 1, the empirical mean
\begin{align*}
\frac{\sum x_i}{n} \geq \frac{1}{n} \geq \frac{1}{10n^2} + \frac{1}{3n} > \mu + \sigma
\end{align*}
The probability that any sample is $1$ is
\begin{align*}
\Pr(\mathrm{any\ sample} = 1) =1 - (1-p)^n \approx pn = \frac{1}{10n}
\end{align*}
In this case, $\frac{1}{\delta}$ is tight.

\subsection{Can we do better?}\label{sec:median}

\paragraph{How about median? }
How far can median of $x$ be from mean of $x$ (in terms of $\sigma$)? By Chebyshev inequality, we have
\begin{align*}
\Pr[|x-\mu| \ge \sqrt{2}\sigma] \le \frac{1}{2}.
\end{align*}
This means more than half of the samples fall into the $\pm \sqrt{2}\sigma$ range, and so does the median.

How many samples do we need if we use median? Let
\[ Y_i = \begin{cases} 
      1 & if\ |x_i-\mu| \le 2\sigma  \\
      0 & if\ |x_i-\mu| \ge 2\sigma
   \end{cases}
\]

By Chebyshev's inequality we get $\Pr[Y_i=1] \ge \frac{3}{4}$.  Note that if $\sum Y_i \geq \frac{n}{2}$, $|\mathrm{median}(x_i)- \mu | \le 2\sigma $.

\begin{align*}
\Pr\left(\sum Y_i \ge \frac{n}{2}\right) &\ge 1 - \Pr\left(\sum Yi \le \E[\sum Y_i] - \frac{n}{4}\right) \\
&\ge 1 - e^{-\frac{2(\frac{n}{4})^2}{n}} \\
&= 1 - e^{-\frac{n}{8}}
\end{align*}

We thus need only
\begin{align*}
n = 8\log \frac{1}{\delta}
\end{align*}
samples to get an estimation within $\mu \pm O(\sigma)$ with probability $1-\delta$. 

\paragraph{Combining mean and median: }
Suppose we divide the samples into $S_1,S_2,\ldots, S_m$ buckets. 
For each bucket, we have $\mathrm{mean}(S_i) = \mu$ and standard deviation $\sigma_{S_i}$ =  $\frac{\sigma}{\sqrt{|S_i|}}$. Let
$|S_i| = \frac{4}{\eps^2} $, we get $\sigma_{s_i} =  \frac{\eps \sigma}{2}$. If we take median of $m = 8 \log\frac{1}{\delta}$ bucket means, according to Section \ref{sec:median}, we get estimation within $\pm \eps \sigma$. Overall we need only
\begin{align*}
|S_i| * m = \frac{4}{\eps^2} * 8 \log\frac{1}{\delta} = 32 \frac{1}{\eps^2} \log \frac{1}{\delta}
\end{align*}
samples. 

\section{Subgaussian random variables}

Recall in last class we see $x$ is subguassian with parameter $\sigma$ if any of following holds.

\begin{enumerate}
\item $\E[e^{\lambda x}] \le e^{\frac{\lambda^2\sigma^2}{2}}$
\item $\Pr[|x|\ge t] \le 2e^{-\frac{t^2}{2\sigma^2}}$
\item $\E[|X|^k]^{\frac{1}{k}} \le \sigma \sqrt{k}$
\end{enumerate}

In fact, these three are equivalent up to scaling $\sigma$ by a constant.

\subsection{Proof (1) $\Rightarrow$ (2)}
\begin{align*}
Pr[X \ge t] &= \Pr[e^{\lambda x} \ge e^{\lambda t}] \\
&\le \frac{E[e^{\lambda x}]}{e^{\lambda t}} \\
&\le e^{\frac{\lambda^2 \sigma^2}{2} - \lambda t} \\
&= e^{\frac{-t^2}{2\sigma^2}}
\end{align*}

The inequality is by Markov's inequality, and the last equality is obtained by choosing $\lambda = \frac{t}{\sigma^2}$. 

\subsection{Proof (2) $\Rightarrow$ (3)}
Let $z = |x|^k$ and assume $\sigma = \frac{1}{\sqrt{2}}$, 
\begin{align}
\E(z) &= \int_{0}^{\infty} \Pr(z \ge \mu) d\mu, \forall z \ge 0. \label{s1} \\ 
\Pr\left( |x|^k \geq t^k \right) &= \Pr\left(|x| \geq t \right) \\
&\leq 2e^{-t^2} \label{s2} \\
\E(|x|^k)&\le \int_{0}^{\infty} 2e^{-t^2} kt^{k-1} dt  \label{s3}  
\end{align}

Equation \ref{s2} is by Chernoff bound, and Equation \ref{s3} is obtained by combining Equation \ref{s1} and Equation \ref{s2}. After some integration by parts, we can get 
\begin{align*}
\E(|x|^k) &\le 2(\frac{k}{2})! \\
&\le 2(\frac{k}{2})^{\frac{k}{2}}
\end{align*}

\subsubsection{Proof (3) $\Rightarrow$ (1)}
By Taylor expansion, 
\begin{align*}
e^{\lambda x} &= 1 + \lambda x + \frac{\lambda^2x^2}{2} + \ldots \\
\E[e^{\lambda x}] &\le 1 + \sum_{k=2}^{\infty} \frac{\lambda^k(ck)^{\frac{k}{2}}}{k!}\\
&\le 1 + \sum_{k = 2}^{\infty} (\frac{\lambda^2 c k e^2}{k^2})^\frac{k}{2} \\
&\le 1 + \sum_{k=2}^{\infty} (\frac{\lambda^2 e}{k})^\frac{k}{2}
\end{align*}

Here we use the fact $k! \ge \left(\frac{k}{e}\right)^k$

for $\sigma = O(1)$
\begin{align*}
e^{\lambda^2\sigma^2} &= 1 + \sum_{k=1}^{\infty} \frac{(\lambda^2\sigma^2)^k}{k!}  \\
&\ge 1 + \sum_{k=1}^{\infty} (\frac{\lambda^2 \sigma^2}{k})^k \\
&= 1 + \sum_{k \in 2\mathbb{N}} (\frac{2\lambda^2 \sigma^2}{k})^{\frac{k}{2}}
\end{align*}

for $\sigma = \Omega(1)$ we have all even terms
\begin{align*}
(\frac{\sqrt{e}\lambda}{\sqrt{k}})^k &\le (\frac{\sqrt{e}\lambda}{\sqrt{k}})^{k-1} + (\frac{\sqrt{e}\lambda}{\sqrt{k}})^{k+1} \\
&\le (\frac{\sqrt{e}\lambda}{\sqrt{k-1}})^{k-1} + (\frac{\sqrt{e}\lambda}{\sqrt{k+1}})^{k+1}  (\sqrt{\frac{k+1}{k}})^{k+1} \\
\end{align*}


\subsection{Subexponential random variables}
Random variable $x$ is subexponential with parameter $\sigma$ if one of the following holds: 
\begin{enumerate}
\item $E[e^{\lambda x}] \le e^{\frac{\lambda^2\sigma^2}{2}}, \forall |\lambda| \le \frac{1}{\sigma}$
\item $Pr[|x|\ge t] \le 2e^{-\frac{t}{2 \sigma}}$
\item $E[|X|^k]^{\frac{1}{k}} \le \sigma k$
\end{enumerate}

\subsection{Subgamma random variables}
Here we state some of the properties of subgamma random variables without proof. Random variable $x$ is $\mathrm{subgamma}(\sigma, B)$ if\\
\begin{align*}
\E[e^{\lambda x}] \le e^{\frac{\lambda^2\sigma^2}{2}},\ \forall |\lambda| \le B \label{subgamma1}
\end{align*}
or equivalently, 
\begin{align}
\Pr[|x|\ge t] \leq
\begin{cases}
2e^{-\frac{t^2}{\sigma^2}} & \text{, if $\frac{|t|}{\sigma^2} \le B$} \\
e^{\frac{1}{2} B^2 \sigma^2 - Bt}\le e^{-\frac{Bt}{2}} & \text{, otherwise}
\end{cases}
\end{align}
Combining the two cases:
\begin{align*}
\Pr[|x|\ge t] \le \max(2e^{-\frac{t^2}{\sigma^2}}, 2e^{-\frac{Bt}{2}})
\end{align*}

Suppose $x_i$ are $\mathrm{subgamma}(\sigma_i, \beta_i)$. Then $\sum x_i$ is $\mathrm{subgamma}(\sqrt{\sum \sigma_i^2}, \min(B_i))$.



\bibliographystyle{alpha}

\begin{thebibliography}{42}

 \bibitem[MR]{MR}
 Rajeev~Motwani, Prabhakar~Raghavan
 \newblock Randomized Algorithms.
 \newblock {\em Cambridge University Press}, 0-521-47465-5, 1995. %pub - isbn - year

%@book{Motwani:1995:RA:211390,
% author = {Motwani, Rajeev and Raghavan, Prabhakar},
% title = {Randomized Algorithms},
% year = {1995},
% isbn = {0-521-47465-5, 9780521474658},
% publisher = {Cambridge University Press},
% address = {New York, NY, USA},
%}


\end{thebibliography}

\end{document}