\documentclass[11pt]{article}
\usepackage{amsmath,amssymb,amsthm,bbm}
\usepackage{tkz-berge}

\DeclareMathOperator*{\E}{\mathbb{E}}
\let\Pr\relax
\DeclareMathOperator*{\Pr}{\mathbb{P}}

\newcommand{\eps}{e}
\newcommand{\inprod}[1]{\left\langle #1 \right\rangle}
\newcommand{\R}{\mathbb{R}}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {\bf CS 388R: Randomized Algorithms } \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[4]{\handout{#1}{#2}{#3}{Scribe: #4}{Lecture #1}}

\newtheorem{theorem}{Theorem}[subsection]
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{observation}[theorem]{Observation}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{property}[theorem]{Property}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{rules}[theorem]{Rule}
\newtheorem{assumption}[theorem]{Assumption}

% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex

\begin{document}

\lecture{23 --- November 25, 2015}{Fall 2015}{Prof.\ Eric Price}{John Kallaugher, Surbhi Goel}

\section{Overview}

This lecture is about Markov Chains, a type of stochastic process where the distribution of the process at time $t$ depends only on the value of the process at time $t-1$.

\section{Introduction}
\begin{definition}[Markov Chain]
A Markov Chain $(X_t)_{t \in \mathbb{N}}$ is a sequence of random variables on some state space $S$ which obeys the following property:

\begin{equation*}
\forall t > 0, (s_i)_{i=0}^t \in S, \mathbb{P}\left\lbrack X_t = s_t \middle| \bigcap_{i=0}^{t-1}(X_i = s_i)\right\rbrack = \mathbb{P}\left\lbrack X_1 = s_t | X_0 = s_{t-1}\right\rbrack
\end{equation*}
\end{definition}

We will write these probabilities as a \emph{transition matrix} $P$, where $P_{ij} = \mathbb{P}\left\lbrack X_1 = s_j | X_0 = s_{i}\right\rbrack$. Note that $\forall i, \sum_j P_{ij} = 1$ is necessary for $P$ to be a valid transition matrix.

If $q \in \mathbb{R}^{|S|}$ is the distribution of $X$ at time 0, the distribution of $X$ at time $t$ will then be $qP^t$.

\subsection{Example: Random Walk on a Graph}
Let our state space be the vertices of a graph $G = (V,E)$. Then we can define a Markov chain by a random walk on $G$, where at each step the walk jumps to a random neighbour of the current vertex. This gives us the following transition matrix:

\begin{equation*}
P_{uv} = \begin{cases}
\frac{1}{d(u)} & (u,v) \in E\\
0 & \mbox{Otherwise.}
\end{cases}
\end{equation*}

\section{The Fundamental Theorem of Markov Chains}
\begin{definition}[Ergodicity]
A Markov Chain is ergodic if $\exists \Pi \in \mathbb{R}^{|S|}$ such that:

\begin{align*}
\forall s \in S, \Pi_s > 0\\
\lim_{t \rightarrow \infty} qP^t = \Pi
\end{align*}
\end{definition}

We will call this $\Pi$ the \emph{stationary distribution} of $X$. Note that when it exists, $\Pi$ is the unique vector $\Pi \in \mathbb{R}^{|S|}$ such that $\Pi P = \Pi$, with $\sum_{s \in S} \Pi_s = 1$ and $\Pi_s \in \lbrack 0,1\rbrack$ for all $s$.

\begin{theorem}[The Fundamental Theorem of Markov Chains]
Let $X$ be a Markov Chain on a finite state space $S = \lbrack n\rbrack$ satisfying the following conditions:

\begin{description}
\item[Irreducibility] There is a path between any two states which will be followed with $> 0$ probability, i.e. $\forall i,j \in \lbrack n\rbrack$, $\exists t \mathbb{P}\lbrack X_t = j | X_0 = i\rbrack > 0$.
\item[Aperiodicity] Let the \emph{period} of a pair of states $u,v$ be the GCD of the length of all paths between them in the Markov chain, i.e. $\gcd\lbrace t \in \mathbb{N}_{>0} | \mathbb{P}\lbrack X_t = v | X_0 = u \rbrack > 0\rbrace$. $X$ is aperiodic if this is 1 for all $u,v$.
\end{description}

Then $X$ is ergodic.
\end{theorem}

Note that both these conditions are necessary as well as sufficient.

\subsection{Further Definitions}
\begin{equation*}
N(i,t) = |\lbrace t \in \mathbb{N} | X_t = i\rbrace|
\end{equation*}

This obeys $\lim_{t \rightarrow \infty} \frac{N(i,t)}{t} = \Pi_i$ for an ergodic chain with stationary distribution $\Pi$.

\begin{equation*}
h_{u,v} = \mathbb{E}\lbrack \min_t \lbrace t | X_t = v\rbrace | X_0 = u\rbrack
\end{equation*}
This is called the \emph{hitting time} of $v$ from $u$, and it obeys $h_{i,i} = \frac{1}{\Pi_i}$ for an ergodic chain with stationary distribution $\Pi$.

\section{Random Walks on Undirected Graphs}
We consider a random walk $X$ on a graph $G$ as before, but now with the assumption that $G$ is undirected.

Clearly, $X$ will be irreducible iff $G$ is connected. It can also be shown that it will be aperiodic iff $G$ is not bipartite. The $\Rightarrow$ direction follows from the fact that paths between two sides of a bipartite graph are always of even length, whereas the $\Leftarrow$ direction follows from the fact that a non-bipartite graph always contains a cycle of odd length.

We can always make a walk on a connected graph ergodic simply by adding self-loops to one or more of the vertices.

\subsection{Ergodic Random Walks on Undirected Graphs}
\begin{theorem}
If the random walk $X$ on $G$ is ergodic, then its stationary distribution $\Pi$ is given by $\forall v \in V, \Pi_v = \frac{d(v)}{2m}$.
\end{theorem}
\begin{proof}
Let $\Pi$ be as defined above. Then:

\begin{align*}
(\Pi P)_v & = \sum_{u,v \in E}\Pi_u \frac{1}{d(u)}\\
& = \sum_{u, u,v \in E}\frac{1}{2m}\\
& = \frac{d(v)}{2m}\\
& = \Pi_v
\end{align*}

So as $\sum_v \Pi_v = \frac{2m}{2m} = 1$, $\Pi$ is the stationary distribution of $X$.
\end{proof}

In general, even on this subset of random walks, the hitting time will not be symmetric, as will be shown in our next example. So we define the commute time $C_{u,v} = h_{u,v} + h_{v,u}$.

\subsection{Example: The Lollipop Graph}
\begin{figure}[h]
        \caption{The Lollipop Graph on 14 Vertices}
        \centering
        \begin{tikzpicture}[%
                        VertexStyle/.style={shape=circle, shading=ball, ball color=blue!60,
                        minimum size=16pt, inner sep=0pt, draw}]
                        \SetVertexNoLabel
                        \grComplete[RA=1]{7}
                        \grPath[x=1,RA=1]{8}
                \end{tikzpicture}
        \end{figure}

        The lollipop graph on $n$ vertices is a clique of $\frac{n}{2}$ vertices connected to a path of $\frac{n}{2}$ vertices. Let $u$ be any vertex in the clique that does not neighbour a vertex in the path, and $v$ be the vertex at the end of the path that does not neighbour the clique. Then $h_{u,v} = \theta(n^3)$ while $h_{v,u} = \theta(n^2)$. This is because it takes $\theta(n)$ time to go from one vertex in the clique to another, and $\theta(n^2)$ time to successfully proceed up the path, but when travelling from $u$ to $v$ the walk will fall back into the clique $\theta(1)$ times as often as it makes it a step along the path to the right, adding an extra factor of $n$ to the hitting time.

\section{Electrical Resistance and Commute Time of a Graph}
View graph $G$ as an electrical network with unit resistors as edges. Let $R_{u,v}$ be the effective resistance between vertices $u$ and $v$. The commute time between $u$ and $v$ in a graph is related to $R_{u,v}$ by $C_{u,v} = 2mR_{u,v}$. We get the following inequalities assuming this relation. 

If $(u,v) \in E$,
\begin{equation*}
R_{u,v} \leq 1 \therefore  C_{u,v} \leq 2m
\end{equation*}

In general, $\forall\ u,v \in V$,
\begin{equation*}
R_{u,v} \leq n-1 \therefore C_{u,v} \leq 2m(n-1) < n^3
\end{equation*}

We inject $d(v)$ amperes of current into $\forall v \in V$. Subsequently we pick some vertex $u \in V$ and remove $2m$ current from $u$ leaving net $d(u) - 2m$ current at $u$. Now we get voltages $x_v$  $\forall v \in V$. Assume we have $x_v -x_u = h_{v,u}$ $\forall v \neq u \in V$ (will prove subsequently). Let $L$ be the Laplacian for $G$ and $D$ be the degree vector, then we have
\begin{equation*}
L x = i_u = D - 2m \mathbbm{1}_u
\end{equation*}
\begin{equation}
\forall v \in V, \sum_{(u,v) \in E} x_v - x_u = d(v)
\end{equation}

\subsection{Lollipop Graph}
Let us revisit the lollipop graph with the electrical network view and compute $h_{u,v}$ and $h_{v,u}$ with $u$ and $v$ as before. To compute $h_{u,v}$. Let $u'$ be the vertex common to the clique and the path. Clearly, the path has resistance $\theta(n)$. $\theta(n)$ current is injected in the path and $\theta(n^2)$ current is injected in the clique.

Consider draining current from $v$. The current in the path is $\theta(n^2)$ as $2m -1 = \theta(n^2)$ current is drained from $v$ which enters $v$ through the path implying $x_u' - x_v = \theta (n^3)$ using Ohm's law ($V=IR$). Now consider draining current from $u$ instead. The current in the path is now $\theta(n)$ implying $x_v - x_u' = \theta(n^2)$ by the same argument.

Since the effective resistance between any edge in the clique is less than 1 and $\theta(n^2)$ current is injected, there can be only $\theta(n^2)$ voltage gap between any 2 vertices in the clique. We get $h_{u,v} = x_u - x_v = \theta(n^3)$ in the former case and $h_{v,u} = x_v - x_u = \theta(n^2)$ in the latter.

\subsection{Proof of Relation}
Define $h'_{v,u} = h_{v,u}$ when $v \neq u$ except $h'_{v,v} = 0$. By current conversion, $\forall u \neq v \in V$, we have
\begin{align*}
h'_{v,u} &= \sum_{(v,w) \in E} \frac{1}{d(v)} (1 + h'_{w,u}) \\
h'_{v,u} &= 1 + \sum_{(v,w) \in E} \frac{1}{d(v)} h'_{w,u}
\end{align*}
\begin{equation}
d(v) = \sum_{(v,w) \in E} h'_{v,u} - h'_{w,u}
\end{equation}

Equations 1 and 2 are linear systems with unique solutions and are identical under $x_v - x_u = h'_{v,u}$ (up to same additive shift to each entry). $x_v = h'_{v,u}$ if $x_u = 0$.

We have shown that for $i_u = D - 2m \mathbbm{1}_u$ with $x = L^{+}i_u$ that $x_v - x_u = h_{v,u}$. For $u'$, we have $x' = L^{+}i_{u'}$. Now, we have,
\begin{equation*}
x - x' = L^{+}(i_u - i_{u'}) = 2m L^{+}(e_{u'} - e_u)
\end{equation*}
The above is equivalent to $2m$ times voltage obtained if you inject 1 ampere at $u'$ and remove 1 ampere from $u$. Using Kirchoff's law we get
\begin{align*}
2m R_{u,u'} &= (x-x')_{u'} - (x-x')_{u} \\
			&= (x_{u'} - x_u) - (x'_u - x'_{u'}) \\
            &= h_{u',u} + h_{u,u'} = C_{u,u'}
\end{align*}

\section{Cover Time of a Graph}
We define $C_u(G)$ as the expected time for a random walk starting at $u$ to visit all vertices in a graph. $C(G)$ is the maximum of $C_u(G)$ over all $u \in V$. 

We have $\forall u \in V$,
\begin{equation*}
C_u(G) \leq 2m(n-1)
\end{equation*}

Consider the spanning tree $T$ of graph $G$. The cover time is bounded by traversing the edges of the tree in both directions (as we could just do a DFS on the spanning tree), and hitting time gives the expected time of moving along an edge, we get
\begin{align*}
C_u(G)	&\leq \sum_{(u,v) \in E(T)} h_{u,v} + h_{v,u}\\
		&= \sum_{(u,v) \in E(T)} C_{u,v} \\
        & \leq (n-1) \max_u C_{u,v} \\
        & \leq 2m(n-1)
\end{align*}
This above inequality is tight for lollipop ($\theta(n^3)$) but not for cliques which has $O(n\log n)$ as we can model it as a coupon collector problem.

Let $R_max = \max_{u,v \in V} R_{u,v}$. We give a tighter bound without proof on $C(G)$ as follows:
\begin{equation*}
mR_{max} \leq C(G) \lesssim mR_{max}\log n
\end{equation*}
\begin{thebibliography}{42}

 \bibitem[MR]{MR}
 Rajeev~Motwani, Prabhakar~Raghavan
 \newblock Randomized Algorithms.
 \newblock {\em Cambridge University Press}, 0-521-47465-5, 1995. %pub - isbn - year

%@book{Motwani:1995:RA:211390,
% author = {Motwani, Rajeev and Raghavan, Prabhakar},
% title = {Randomized Algorithms},
% year = {1995},
% isbn = {0-521-47465-5, 9780521474658},
% publisher = {Cambridge University Press},
% address = {New York, NY, USA},
%}
\end{thebibliography}

\end{document}