lecture 22

inputs/lecture_22.tex

@@ -5,66 +5,65 @@
     We want to start with the basics of the theory of Markov chains.
 \end{goal}
 
-% \begin{example}[Markov chains with two states]
+ \begin{example}[Markov chains with two states]
-%     Suppose there are two states of a phone line,
+     Suppose there are two states of a phone line,
-%     $0$,``free'', or $1$, ``busy''.
+     $0$,``free'', or $1$, ``busy''.
-%     We assume that the state only changes at discrete units of time
+     We assume that the state only changes at discrete units of time
-%     and model this as a sequence of random variables.
+     and model this as a sequence of random variables.
-%     Assume
+     Assume
-%     \begin{IEEEeqnarra*}{rCl}
+     \begin{IEEEeqnarray*}{rCl}
-%         \bP[X_{n+1} = 0 | X_n = 0] &=& p\\
+         \bP[X_{n+1} = 0 | X_n = 0] &=& p\\
-%         \bP[X_{n+1} = 0 | X_n = 1] &=& (1-p)\\
+         \bP[X_{n+1} = 0 | X_n = 1] &=& (1-p)\\
-%         \bP[X_{n+1} = 1 | X_n = 0] &=& q\\
+         \bP[X_{n+1} = 1 | X_n = 0] &=& q\\
-%         \bP[X_{n+1} = 1 | X_n = 1] &=& (1-q)
+         \bP[X_{n+1} = 1 | X_n = 1] &=& (1-q)
-%     \end{IEEEeqnarra*}
+     \end{IEEEeqnarray*}
-%     for some $p,q \in (0,1)$.
+     for some $p,q \in (0,1)$.
-%     We can write this as a matrix
+     We can write this as a matrix
-%     \begin{IEEEeqnarra*}{rCl}
+     \begin{IEEEeqnarray*}{rCl}
-%         P &=& \begin{pmatrix}
+         P &=& \begin{pmatrix}
-%             p & (1-p) \\
+             p & (1-p) \\
-%             q & (1-q)
+             q & (1-q)
-%         \end{pmatrix} 
+         \end{pmatrix}
-%     \end{IEEEeqnarra*}
+     \end{IEEEeqnarray*}
-%     Note that the rows of this matrix sum up to $1$.
+     Note that the rows of this matrix sum up to $1$.
-% 
+
-%     Additionally, we make the following assmption:
+     Additionally, we make the following assmption:
-%     Given that at some time $n$, the phone is in state $i \in \{0,1\}$,
+     Given that at some time $n$, the phone is in state $i \in \{0,1\}$,
-%     the behavior of the phone after time $n$ does not depend
+     the behavior of the phone after time $n$ does not depend
-%     on the way, the phone reached state $i$.
+     on the way, the phone reached state $i$.
-% 
+
-%     \begin{question}
+     \begin{question}
-%         Suppose $X_0 = 0$.
+         Suppose $X_0 = 0$.
-%         What is the probability, that the phone will be free at times
+         What is the probability, that the phone will be free at times
-%         $1 \& 2$ and will become busy at time $3$,
+         $1 \& 2$ and will become busy at time $3$,
-%         i.e.~what is  $\bP[X_1 = 0, X_2 = 0, X_3 = 1]$?
+         i.e.~what is  $\bP[X_1 = 0, X_2 = 0, X_3 = 1]$?
-%     \end{question}
+     \end{question}
-%     We have
+     We have
-%     \begin{IEEEeqnarra*}{rCl}
+     \begin{IEEEeqnarray*}{rCl}
-%         \bP[X_1 = 0, X_2 = 0, X_3 = 1]
+         \bP[X_1 = 0, X_2 = 0, X_3 = 1]
-%         &=& \bP[X_3 = 0 | X_2 = 0, X_1 = 0] \bP[X_2 = 0, X_1 = 0]\\
+         &=& \bP[X_3 = 0 | X_2 = 0, X_1 = 0] \bP[X_2 = 0, X_1 = 0]\\
-%         &=& \bP[X_3 = 0 | X_2 = 0] \bP[X_2 = 0, X_1 = 0]\\
+         &=& \bP[X_3 = 0 | X_2 = 0] \bP[X_2 = 0, X_1 = 0]\\
-%         &=& \bP[X_3 = 0 | X_2 = 0] \bP[X_2 = 0 |  X_1 = 0] \bP[X_1 = 0]\\
+         &=& \bP[X_3 = 0 | X_2 = 0] \bP[X_2 = 0 |  X_1 = 0] \bP[X_1 = 0]\\
-%         &=& P_{0,1} P_{0,0} P_{0,0}
+         &=& P_{0,1} P_{0,0} P_{0,0}
-%     \end{IEEEeqnarra*}
+     \end{IEEEeqnarray*}
-% 
+
-%     \begin{question}
+     \begin{question}
-%         Assume $X_0 = 0$. What is $\bP[X_3 = 1]$?
+         Assume $X_0 = 0$. What is $\bP[X_3 = 1]$?
-%     \end{question}
+     \end{question}
-%     For $\{X_3 = 1\}$ to happen, we need to look at the following
+     For $\{X_3 = 1\}$ to happen, we need to look at the following
-%     disjoint events:
+     disjoint events:
-% %    \begin{IEEEeqnarra*}{rCl}
+    \begin{IEEEeqnarray*}{rCl}
-% %        \bP(\{X_3 = 1, X_2 = 0, X_1 = 0\}) &=& P_{0,1} P_{0,0}^2,\\
+        \bP(\{X_3 = 1, X_2 = 0, X_1 = 0\}) &=& P_{0,1} P_{0,0}^2,\\
-% %        \bP(\{X_3 = 1, X_2 = 0, X_1 = 1\}) &=& P_{0,1}^2 P_{1,0},\\
+        \bP(\{X_3 = 1, X_2 = 0, X_1 = 1\}) &=& P_{0,1}^2 P_{1,0},\\
-% %        \bP(\{X_3 = 1, X_2 = 1, X_1 = 0\}) &=& P_{0,0} P_{0,1} P_{1,1},\\
+        \bP(\{X_3 = 1, X_2 = 1, X_1 = 0\}) &=& P_{0,0} P_{0,1} P_{1,1},\\
-% %        \bP(\{X_3 = 1, X_2 = 1, X_1 = 1\}) &=& P_{0,1} P_{1,1}^2.
+        \bP(\{X_3 = 1, X_2 = 1, X_1 = 1\}) &=& P_{0,1} P_{1,1}^2.
-% %    \end{IEEEeqnarr*}
+    \end{IEEEeqnarray*}
-% 
+
-%     More generally, consider a Matrix $P \in  (0,1)^{n \times n}$ 
+     More generally, consider a Matrix $P \in  (0,1)^{n \times n}$
-%     whose rows sum up to $1$.
+     whose rows sum up to $1$.
-%     Then we get a Markov Chain with $n$ states
+     Then we get a Markov Chain with $n$ states
-%     by defining $\bP[X_{n+1} = i | X_{n} = j] = P_{i,j}$.
+     by defining $\bP[X_{n+1} = i | X_{n} = j] = P_{i,j}$.
-%     
+ \end{example}
-% \end{example}
 
 \begin{definition}
     Let $E$ denote a \vocab{discrete state space},
@@ -117,15 +116,15 @@
     Consider
     \begin{IEEEeqnarray*}{rCl}
         P &=&
-        \begin{pmatrix} 
+        \begin{pmatrix}
             & \ddots & \ddots & \ddots &  & & & & 0\\
             \ldots & 0 & \frac{1}{2} & 0 & \frac{1}{2} & 0 & \ldots \\
             & \ldots & 0 & \frac{1}{2} & 0 & \frac{1}{2} & 0 & \ldots \\
             & & \ldots & 0 & \frac{1}{2} & 0 & \frac{1}{2} & 0 & \ldots \\
             0 & & & & & \ddots & \ddots & \ddots & \\
-        \end{pmatrix} 
+        \end{pmatrix}
     \end{IEEEeqnarray*}
-    
+
 \end{example}
 
 % \begin{example}
@@ -144,9 +143,9 @@
     For every $x \in E$,
     let $\mathbf{P}(x, \cdot )$ be a probability measure on $E$.%
     \footnote{$\mathbf{P}(x,\cdot )$ corresponds to a row of our matrix in the discrete case}
-   
+
     Given the triples $(E, \alpha, \{\mathbf{P}(x, \cdot )\}_{x \in E})$,
-    we say that a stochastic process $(X_n)_{n \ge 0}$ 
+    we say that a stochastic process $(X_n)_{n \ge 0}$
     is a \vocab[Markov chain]{Markov chain taking values on $E$ %
         with starting distribution $\alpha$ %
         and transition probability $\{\mathbf{P}(x, \cdot )\}_{x \in E}$}
@@ -157,9 +156,9 @@
             \[
             \bE[f(X_{n+1}) | \cF_n] = \bE[f(X_{n+1}) | X_n]
             = \int_E f(y) \mathbf{P}(X_n, \dif y) \text{ a.s.}
-            \] 
+            \]
     \end{enumerate}
-    
+
 \end{definition}
 \begin{remark}
     This agrees with the definition in the discrete case,
@@ -170,49 +169,44 @@
 \begin{notation}
     If $\{\mathbf{P}(x, \cdot )\}_{x \in E}$ is a transition probability,
     then for all $f: E \to  \R$ bounded and measurable,
-    define  $\mathbf{P} : \cB_{\text{bdd}}(E) \to  \cB_{\text{bdd}}$ 
+    define  $\mathbf{P} : \cB_{\text{bdd}}(E) \to  \cB_{\text{bdd}}$
     by
     \[
         (\mathbf{P} f)(x) \coloneqq \int_E f(y) \mathbf{P}(x, \dif y).
-    \] 
+    \]
 \end{notation}
 We get the following fundamental link between martingales and Markov chains:
 \begin{theorem}
     \label{martingalesandmarkovchains}
-    Suppose $(E, \alpha, \{\mathbf{P}(x, \cdot )\}_{x \in E})$ 
+    Suppose $(E, \alpha, \{\mathbf{P}(x, \cdot )\}_{x \in E})$
     is given.
     Then a stochastic process $(X_n)_{n \ge 0}$ is a Markov chain
     iff for every $f: E \to \R$ bounded, measurable,
     \[
     M_n(f) \coloneqq f(X_n) - f(X_0) - \sum_{j=1}^{n-1} (\mathbf{I} - \mathbf{P})f(X_j)
-    \] 
+    \]
     is a martingale
     with respect to the canonical filtration of $(X_n)$.
 \end{theorem}
 \begin{proof}
-    $\implies$ 
+    $\implies$
     Fix some bounded, measurable $f : E \to \R$.
     Then, for all $n$, $M_n(f)$ is bounded
     and hence $M_n(f) \in  L^1$.
     $M_n(f)$ is $\cF_n$-measurable for all $n \in \N$.
-    
-    \begin{claim}
-        $\bE[M_{n+1}(f) | \cF_n] = M_n(f)$.
-    \end{claim}
-    \begin{subproof}
-        It suffices to show
-        $\bE[M_{n+1}(f) - M_n(f) | \cF_n] = 0$ a.s.
 
-        We have
+    In order to prove $\bE[M_{n+1}(f) | \cF_n] = M_n(f)$,
-        \begin{IEEEeqnarray*}{rCl}
+    it suffices to show $\bE[M_{n+1}(f) - M_n(f) | \cF_n] = 0$ a.s.
-            \bE[M_{n+1}(f) - M_n(f) | \cF_n]
-            &=& \bE[f(X_{n+1} | \cF_n] - (\mathbf{P}f)(X_n)\\
-            &\overset{\text{Markov property}}{=}& (\mathbf{P}f)(X_n) - (\mathbf{P}f)(X_n)\\
-            &=& 0
-        \end{IEEEeqnarray*}
-    \end{subproof}
 
-    $\impliedby$ 
+    We have
+    \begin{IEEEeqnarray*}{rCl}
+        \bE[M_{n+1}(f) - M_n(f) | \cF_n]
+        &=& \bE[f(X_{n+1} | \cF_n] - (\mathbf{P}f)(X_n)\\
+        &\overset{\text{Markov property}}{=}& (\mathbf{P}f)(X_n) - (\mathbf{P}f)(X_n)\\
+        &=& 0
+    \end{IEEEeqnarray*}
+
+    $\impliedby$
     Suppose $(M_n(f))_n$ is a martingale for all bounded, measurable $f$.
     By the martingale property, we have
     \begin{IEEEeqnarray*}{rCl}