\documentclass[11pt]{article}
\usepackage{amsmath,amssymb,amsthm}

\DeclareMathOperator*{\E}{\mathbb{E}}
\let\Pr\relax
\DeclareMathOperator*{\Pr}{\mathbb{P}}
\DeclareMathOperator*{\In}{\mathbb{I}}
\DeclareMathOperator*{\Fo}{\mathcal{F}}
\DeclareMathOperator*{\median}{median}

\newcommand{\eps}{\epsilon}
\newcommand{\inprod}[1]{\left\langle #1 \right\rangle}
\newcommand{\R}{\mathbb{R}}
\newcommand{\poly}{{\rm poly}}


\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {\bf CS 395T: Sublinear Algorithms } \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[4]{\handout{#1}{#2}{#3}{Scribe: #4}{Lecture #1 }}

\newtheorem{thm}{Theorem}[section]
\newtheorem{claim}[thm]{Claim}
\newtheorem{lemma}[thm]{Lemma}
\newtheorem{define}[thm]{Definition}
\newtheorem{cor}[thm]{Corollary}
\newtheorem{obs}[thm]{Observation}
\newtheorem{remark}[thm]{Remark}
\newtheorem{example}[thm]{Example}
\newtheorem{construction}[thm]{Construction}
\newtheorem{conjecture}{Conjecture}
\newtheorem{THM}{Theorem}
\newtheorem{COR}[THM]{Corollary}
\newtheorem{question}{Question}
\newtheorem{fact}[thm]{Fact}
\newtheorem{prop}[thm]{Proposition}
% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex

\begin{document}

\lecture{5 --- Sept. 11, 2014}{Fall 2014}{Prof.\ Eric Price}{Alex Knaust}

In today's lecture, we will cover the following topics:

\begin{enumerate}
\item Complete our analysis of count-sketch point query \cite{MP2014}
\item An algorithm with $O(k\log^2 n)$ recovery \emph{time} due to \cite{GLPS2012}
\end{enumerate}

\section{Count-sketch Analysis Continued}
Recall we had the following definitions
\[ h_u: [n] \to [B] \quad \text{\parbox{5cm}{A family of independent hash functions}}\]
\[ s_u: [n] \to {+1, -1} \quad \text{\parbox{5cm}{A family of random sign functions}}\]
A hashtable $Y$ with $R$ rows and $B$ columns, being used as follows:
\[ Y_{u,v} = \sum_{i:h(i) = v} s_u(i)x_i \]
\[\hat{x_i}^{(u)} = s_u(i)Y_{u,h(i)} \]
\[\hat{x_i} = \median_{u\in [R]} \hat{x_i}^{(u)}\]
\[ \hat{x_i} = s_u(i)Y_{u,h(i)} \]

We would like to estimate the error 
\[\Delta_i^u = \hat{x_i}^{(u)} - x_i\]
Which we can rewrite as 
\[\Delta_i^u = \sum_{j\neq i} {\underbrace{s_u(j)x_j\In_{h_u(i) = h_u(j)}}_{z_{u,j}}}\]
Splitting into the largest coordinates $H = (1...k)$ and the rest $T = (k+1...n)$
\[\Delta_i^u = \underbrace{\sum_{H\setminus\{i\}}{z_{u,j}}}_{=0 \text{ with prob. .9}} + \sum_{T\setminus\{i\}}{z_{u,j}}\]
and by the same argument from lecture 4 (cross terms cancelling due to $s_u$ being independent)
\[\E\left[\left(\sum_{T\setminus\{i\}}{z_{u,j}}\right)^2\right] \leq \frac{||x_T||_2^2}{B}\]
\[\Rightarrow |\Delta_i^u| \leq \frac{||x_T||_2^2}{k} \quad \text{with $\frac{4}{5}$ prob.}\]
\[\Rightarrow |\Delta_i| = |x - \hat{x_i} | \leq \frac{||x_T||_2^2}{k} \quad \text{with $1 - e^{-\Omega(R)}$ prob.}\]

\subsection{Using the Fourier Transform}
Recall for a symmetric random variable we defined the Fourier transform as
\[\Fo_X(t) = \E_{x \sim X}[\cos{(2\pi xt)}]\] 
For $z_{u,i}$ we then have 
\[z_{u,i} = s_u(j)x_j\In_{h_u(i) = h_u(j)}\]
Which is 0 (no collision) with prob. $1-\frac{1}{B}$, and $\pm x_i$ with prob. $\frac{1}{2B}$
\[\E[\cos(2\pi z_{u,i} t)] = \left(1 - \frac{1}{B}\right)\cos{0} + \frac{1}{B}\cos(2\pi tx_i)\]
\[\geq\left(1 - \frac{2}{B}\right) \geq 0 \]
Furthermore, since adding PDFs is equivalent to convolving them, we can write
\[\Fo_{\sum_{T\setminus\{i\}}{z_{u,j}}}(t) = \prod_{j \in T\setminus \{i\}} \Fo_{z_{u,j}}(t) \geq 0\]
Since the sum has a non-negative fourier transform, we can apply our previous lemma (Lemma 3.1 in \cite{MP2014})
\begin{equation}
\label{eq:lemma31use}
\Rightarrow \Pr\left[ \left| \sum_{T\setminus\{i\}}{z_{u,j}} \right|  \leq \epsilon \frac{||x_T||_2}{\sqrt{B}}\right] \gtrsim \epsilon
\end{equation}
The sets $H$ and $T$ are independent, thus

\begin{equation}
\label{eq:boundDiu}
\Pr \left[ |\Delta_i^u| \leq \epsilon\frac{||x_T||_2}{\sqrt{B}} \right] 
\geq 
\underbrace{\Pr\left[ \sum_{H\setminus\{i\}}{z_{u,j}} = 0\right]}_{\text{.9 with prev.}}
\cdot
\underbrace{\Pr\left[ \left|\sum_{T\setminus\{i\}}{z_{u,j}}\right| \leq \epsilon\frac{||x_T||_2}{\sqrt{B}} \right]}_{ \Omega{(\epsilon)} \text { due to Equation \ref{eq:lemma31use}} }
\gtrsim \epsilon
\end{equation}

{\bf Question:} So what happens to the median of the errors, $\Delta_i = \hat{x_i} - x_i = \median_{u} \Delta_i^{u}$?

\begin{lemma}(Lemma 3.3 from \cite{MP2014})
\label{le:medsym}
Let $\Delta_i^u$ for $u \in [R]$ be symmetric independent random variables. And let equation \ref{eq:boundDiu} apply, then

\begin{equation}
\Pr\left[ \left|\median_{u\in [R]}\Delta_i^u\right| \geq \epsilon\frac{||x_T||_2}{\sqrt{B}} \right] < 2e^{-\Omega(R\epsilon^2)}
\end{equation}
\end{lemma}

\begin{proof}
Let $\In_u$ denote the indicator of the event $\Delta_i^u \geq \epsilon\frac{||x_T||_2}{\sqrt{B}}$. These $\In_u$ are bounded and therefore subgaussian, so their sum $\sum_u^R{\In_u}$ is also subgaussian with parameter $\sigma = \frac{\sqrt{R}}{2}$ and mean $\mu = \frac{R}{2}(1 - \Omega(\epsilon))$. Hence using the Chernoff bound (see lecture 1) we have
\begin{equation}
\label{eq:indbound}
\Pr\left[ \sum_u^R \In_u \geq \mu + \Omega(\epsilon R)\right] \leq  e^{\frac{-\Omega(\epsilon R)^2}{R/4}} = e^{-\Omega(2\epsilon ^2 R)}
\end{equation}

Equation \ref{eq:indbound} also applies if $\In_u = \Delta_i^u \leq - \epsilon\frac{||x_T||_2}{\sqrt{B}}$. If neither of the events occurs then the median must lie in $(- \epsilon\frac{||x_T||_2}{\sqrt{B}}, \epsilon\frac{||x_T||_2}{\sqrt{B}})$.
\end{proof}

If we let $\epsilon = \sqrt{\frac{t}{R}}$ and use \ref{le:medsym}, we arrive at 
\begin{equation}
\Pr\left[ \left|\Delta_i\right| \geq \sqrt{\frac{t}{R}}\frac{||x_T||_2}{\sqrt{B}} \right] < 2e^{-\Omega(t)}
\end{equation}


\section{Gilbert-Li-Porat-Strauss Fast Recovery}
The methods we have looked at so far optimize for space only. Gilbert-Li-Porat-Strauss proposed an alternate method in 2009 \cite{GLPS2012} that only take $O(k\log^2n)$ time as well as space.

For their paper, they use the the constraint $||\hat{x} - x||_2 \leq (1 + \epsilon)\underbrace{||x_T||_2}_{= \quad\text{Err}_2(x,k)}$. (Reminder that $x_T$ is the vector $x$ with the $k$ largest elements zeroed out)

In class we prove it for a weaker $L_1$ constraint instead, $||\hat{x} - x||_1 \leq (1 + \epsilon)\text{Err}_1(x,k)$.
\bibliographystyle{alpha}


\begin{define} Let $H$ be the set of "heavy-hitters"
\[H = \left\lbrace i \quad|\quad |x_i| \geq \frac{\text{Err}_1(x,k)}{k} \right\rbrace\]
\end{define}
There can be at most $2k$ heavy hitters $|H| \leq 2k$

It suffices to find a superset $S$ such that $|S| \leq o(k), S \supset H$.
If we had such a set, the we could estimate $x_S$ using count-min and get
\begin{equation}
||\hat{x}_S - x_S||_1 \leq |S|\frac{\text{Err}_1(x,k)}{k} = O\left(\frac{\text{Err}_1(x,k)}{k}\right)
\end{equation}
If we split the error $||\hat{x_S} - x||_1$ into the heavy-hitters and non-heavy hitters that are not in $S$
\begin{equation}
||\hat{x_S} - x||_1 = 
\underbrace{||\hat{x_S} - x_S||_1}_{= O(\text{Err}_1(x,k))} + 
\underbrace{||x_{(\text{top } k)\cap \bar{S}}||_1}_{= k \cdot \frac{\text{Err}_1(x,k)}{k}} + 
\underbrace{||x_{(\text{not top } k)\cap \bar{S}}||_1}_{= \text{Err}_1(x,k)}
= O(\text{Err}_1(x,k))
\end{equation}

Unfortunately, it is still a bit difficult to find such an $S \supset H$. We can at least find a set $S$ that has `most` of the heavy hitters.

\begin{lemma}
In $O(k\log n)$ time and space we can recover
\[S, \quad |S| \leq o(k)\quad \forall i \in H, i\in S \text{ with 4/5 probability}\]
\end{lemma}

\begin{proof}
\textbf{Idea: } Use a hashtable with some clever signing.

Let $h : [n] \to [B]$ be a hash function, and let $c_i = \left\lbrace j \quad | h(j) = h(i), \quad j \neq i \right\rbrace$

We know from previous arguments about the number of heavy hitter collisions in a hashtable that
\begin{equation}
	||x_{c_i}|| \leq \frac{\text{Err}_1(x, k)}{k} \quad \text{with 4/5 probability}
\end{equation}

Also, by definition
\begin{equation}
\frac{\text{Err}_1(x, k)}{k} \leq ||x_i||_1 \quad \forall i \in H
\end{equation}

For each bucket, make $O(\log n)$ measurements that sum the contents with different signs	
\[
\begin{array}{cc}
i = 0 & ++++++++++ \\ 
i = 1 & +++++++++- \\ 
i = 2 & ++++++++-+ \\
\vdots & \vdots
\end{array} 
\]

The signs are the bit representation of the index. i.e.

\[Y_{1, v} = \sum_{h(j) = v} x_j \cdot (-1)^{j \& 1}\]

\[Y_{t, v} = \sum_{h(j) = v} x_j \cdot (-1)^{(j >> t-1) \& 1}\]


If $i$ dominates a bucket:

	\[\text{sign}(Y_{t,h(i)}) = (-1)^{(i >> t -1)\& 1}\quad \forall t \in [R]\]

So we can recover $i$ with good probability.
\end{proof}

For an $L_2$ approximation we can instead use an error correcting code \cite{GLPS2012}

\textbf{Idea: }
If $S$ really contains $H$, we previously showed
\[S \supset H \Rightarrow ||\hat{x}_S - x||_1 \leq (1 + \epsilon) \text{Err}_1(x,k)\]

But $S$ only mostly contains $H$. So instead, the first time we get at least $k/2$ of the top $k$, if we subtract these from $x$ we can try again to get half of the remaining top $k/2$

\[\text{Err}_1 \left(x - \hat{x}_S, \frac{k}{2}\right) \leq (1 + \epsilon) \text{Err}_1(x, k)\]

So in $O(k\log n)$ time and space we get a linear sketch $Ax \to \hat{x}_S$. The trick is to repeatedly perform this algorithm on $\hat{x}_S$. I.e. we compute a new $A'x$ with $k/2$ and also $A'\hat{x}_S$ to get $ A'(x - \hat{x}_S) \to \hat{x}_{S'}$.

\begin{equation}
\label{eq:shrinking}
 \text{Err}_1(x - \hat{x}_S - \hat{x}_{S'}, k/4) \leq (1 + \epsilon)\text{Err}_1(x - \hat{x}_S, k/2) \leq (1 + \epsilon)^2 \text{Err}_1(x,k)
 \end{equation}

Now we can repeatedly perform the algorithm for $i = 1, ..., \log{n}$
\begin{itemize}
	\item Let $k_i = \frac{k}{2^i}$
	\item Let $\epsilon_i = \frac{1}{10}\left(\frac{2}{3}\right)^i$ decay exponentially
	\item Compute $A^{(i)}$ with $k_i$ and $\epsilon_i$
\end{itemize}

Since $\epsilon_i$ are decaying, their sum forms a geometric series, and
\begin{equation}
\label{eq:geometricserie}
\prod_{i = 1}^{\log n} (1 + \epsilon_i) \leq e^{\sum \epsilon_i} = e^{O(1)} = O(1)
\end{equation}

Then using the same argument as in Equation \ref{eq:shrinking}, and Equation \ref{eq:geometricserie}
\begin{equation}
\left\| x - \sum_{i=1}^{\log n} \hat{x}_{S_i}^{(i)} \right\|_1 \leq \prod_{i=1}^{\log n} (1 + \epsilon_i) \text{Err}_1(x,k) \lesssim \text{Err}_1(x, k)
\end{equation}

\textbf{Question: } So what are the total costs of this algorithm?

The space needed to perform this algorithm is $\sum \frac{k_i}{\epsilon_i} \log n = O(k\log n)$

The analysis of the running time has two different parts
\begin{itemize}
	\item The time to do the recovery algorithm :  $O(k \log n)$
	\item The time to perform the subtractions $A'(x - \hat{x}_S)$ : $O(k\log^2 n)$
\end{itemize}

\begin{thebibliography}{42}


\bibitem[GLPS2012]{GLPS2012}
Anna C. Gilbert, Yi Li, Ely Porat, Martin J. Strauss.
\newblock Approximate Sparse Recovery: Optimizing Time and Measurements
\newblock{\em SIAM Journal on Computing} 41(2):436--453, 2012

\bibitem[MP2014]{MP2014}
Gregory T. Minton, Eric Price
\newblock Improved Bounds for Count-Sketch
\newblock{\em Proceedings of the Twenty-Fifth Annual ACM-SIAM Symposium on Discrete Algorithms} 51:669--686, 2014
\end{thebibliography}


\end{document}

