\documentclass[11pt]{article}

% This package contains many of the macros you may need.
%
% A useful environment defined in it is the example environment, which you can
% invoke via \begin{example} and \end{example}.
\usepackage{scribe}
\usepackage{enumerate}
\usepackage{enumitem}

\renewcommand{\d}{\,\mathrm{d}}
\renewcommand{\abs}[1]{\left\lvert#1\right\rvert}
\DeclareMathOperator{\diam}{diam}
\DeclareMathOperator{\dist}{dist}

\begin{document}

\begin{center}
  \fcolorbox{black}{white}{
    \begin{minipage}{.9\columnwidth}
      \textbf{Stats 300b: Theory of Statistics}
      \hfill
      \textbf{Winter 2017} \\
      \vspace{-.2cm}
      % Instead of Lecture 1, January 10, put the actual lecture number and
      % date of the lecture here
      \begin{center}
        {\LARGE Lecture 14 -- February 23} \\
      \end{center}
      \emph{Lecturer: John Duchi}
      \hfill
      \emph{Scribe: Tri Dao}  % You should put your name here if you scribe
    \end{minipage}
  }
\end{center}

\begin{center}
  \dbend ~~  \emph{\textbf{Warning:} these notes may contain factual errors}
\end{center}

\paragraph{Reading:} VDV Chapter 18-19.
%%%%%%%%%%%%%%%%% SCRIBE NOTES HERE %%%%%%%%%%%%%%%%%%%%

\paragraph{Outline:}
Uniform limits in distribution:
\begin{itemize}
  \item Stochastic analog of equicontinuity.
  \item Moduli of continuity of processes
\end{itemize}

\section{Recap: Compactness in functions spaces}

Last time we talked about compactness in $\mathcal{C}(T, \mathbb{R})$ (i.e.,
continuous functions from $T \to \mathbb{R}$).

We defined \emph{modulus of continuity}: $\omega_f(\delta) \defeq \sup_{d(s, t) < \delta} \{
\abs{f(t) - f(s)} \}$.

We also stated (but didn't prove) Arzela--Ascoli theorem: $\mathcal{F} \subseteq
\mathcal{C}(T, \mathbb{R})$ is compact if and only if:
\begin{enumerate}[label=(\roman*)]
  \item $\sup_{f \in \mathcal{F}} \abs{f(t_0)} < \infty$ for some $t_0$.
  \item $\lim_{\delta \downarrow 0} \sup_{f \in \mathcal{F}} \omega_f(\delta) = 0$ (equicontinuity)
  \item $\mathcal{F}$ is closed.
\end{enumerate}

How do we get analog of (ii) for stochastic processes, so that sequences $X_n
\in L_\infty(T)$ take values in compact sets and have limits?

\section{Stochastic analog of equicontinuity}

\begin{definition}
  Let $X_n \in L_\infty(T)$ be random variables.
  We say that $X_n$ are \emph{asymptotically equicontinuous} if for all $\eta,
  \epsilon > 0$, there is a finite partition $T_1, \dots, T_k$ of $T$ such that
  \begin{equation*}
    \limsup_{n \to \infty} \P \left( \max_i \sup_{s, t \in T_i} \abs{X_{n, s} - X_{n, t}}
    \geq \epsilon\right) \leq \eta.
  \end{equation*}
\end{definition}
Recall that we have a sequence $\Omega_n$ of sample spaces, with $X_n \colon \Omega_n \to
L_\infty(T)$, and $X_{n, t}(\omega)$ is the value of $X_n(\omega)$ at $t$.

This definition is saying that for any fix amount of separation, we can divide
$T$ into little blocks so that the process is stochastically continuous within
each block.

\begin{example}
  Let $Z_i \in \mathbb{R}^d$ with $Z_i \simiid P$.
  Assume that $\E[\norm{Z_i}^2] < \infty$ and $\E[Z_i] = 0$.
  Define $X_{n, t} \defeq \frac{1}{\sqrt{n}} \sum_{i=1}^n Z_i^T t$ for $t \in T
  \subseteq \mathbb{R}^d$ (with $T$ compact).
  Hence for any $\delta > 0$,
  \begin{equation*}
    \P \left( \sup_{\norm{t - s} \leq \delta} \abs{X_{n, t} - X_{n, s}} \geq \epsilon\right) = \P
    \left( \norm{\frac{1}{\sqrt{n}} \sum_{i=1}^n Z_i} \delta \geq \epsilon \right) \leq \frac{\E
      \norm{Z_i}^2 \delta^2}{\epsilon^2} \quad \text{by Chebyshev}.
  \end{equation*}
  Hence if we choose $\delta$ small enough, we get $\frac{\delta^2 \E \norm{Z_i}^2}{\epsilon^2}
  \leq \eta$ for any desired $\eta > 0$.

  As $T$ is compact, we have partition of $T$ into finite blocks of radius less
  than $\delta$.
  Hence $X_n$ is a sequence of asymptotically equicontinuous random variables.
\end{example}

With stochastic equicontinuity, we put down finite balls centered at $t^i$ in
our set $T$.
Somehow if $X_{n, t} \approx X_{n, t^i}$ for some $t^i$, then if $X_{n, t^i}$
converges in distribution to something, $X_{n, t}$ should too.

\begin{theorem}
  The following are equivalent:
  \begin{enumerate}[label=(\roman*)]
    \item $X_i \in L_\infty(T)$ and $X_n \cd X \in L_\infty(T)$ where $X$ is tight.
    \item
    \begin{enumerate}
      \item Finite dimensional convergence (FIDI): $(X_{n, t_1}, \dots, X_{n,
        t_k}) \cd $ something for any $t_1, \dots, t_k \in T$, and $k < \infty$.
      \item $X_n$ are (asymptotically) stochastically equicontinuous.
    \end{enumerate}
  \end{enumerate}
\end{theorem}

\begin{proof}
  We only care about $\text{(ii)} \implies \text{(i)}$.

  \paragraph{Part 1: Construct $T_0 \subset T$ a countable dense subset and work there.}
  Let $m \in \mathbb{N}$, construct partitions $T_1^m, \dots, T_{k_m}^m$ of $T$
  such that
  \begin{equation*}
    \limsup_{n \to \infty} \P \left( \max_i \sup_{s, t \in T_i^m} \abs{X_{n, s} - X_{n,
          t}} \geq 2^{-m} \right) \leq 2^{-m}.
  \end{equation*}
  Assume without loss of generality that partitions are nested.

  For each $m$, define
  \begin{equation*}
    \rho_m(s, t) =
    \begin{cases}
      0 &\mbox{if } s, t \in T_i^m \text{ for some } i \\
      1 &\mbox{otherwise}.
    \end{cases}
  \end{equation*}
  Define $\rho(s, t) \defeq \sum_{m = 1}^\infty 2^{-m} \rho_m(s, t)$.
  So then $\rho\text{--}\diam(T_i^m) \leq \sum_{i=1}^{m} 0 + \sum_{i=m+1}^{\infty} 2^{_i} = 2^{-m}$.
  Let $t_{i, m} \in T_i^m$ (arbitrary) and define
  \begin{equation*}
    T_0 \defeq \bigcup_{m=1}^\infty \bigcup_{i=1}^{k_m} \{ t_{i, m} \}
  \end{equation*}
  then $T_0$ is countable and dense in $T$ for $\rho$-metric.
  Moreover, $T$ is totally bounded in the $\rho$-metric.

  \paragraph{Part 2: Use metric $\rho$ to extend process to $\mathcal{C}(T, \mathbb{R})$.}
  By Kolmogorov's extension theorem, there is \emph{some} stochastic process $\{
  X_t \}_{t \in T_0}$ such that
  \begin{equation*}
    (X_{n, t_1}, \dots, X_{n, t_k}) \cd (X_{t_1}, \dots, X_{t_k}) \quad \text{for
      all } t_i \in T_0.
  \end{equation*}
  The standard result on convergence in distribution: for all $S \subset T_0$ with
  $\card(S) < \infty$, we have
  \begin{align*}
    2^{-m}
    &\geq \limsup_{n \to \infty} \P \left( \max_i \max_{\substack{s, t \in T_i^m \\ s, t \in S}}
      \abs{X_{n, t} - X_{n, s}} \geq 2^{-m} \right) \\
    &\geq \P \left( \max_i \max_{\substack{s, t \in T_i^m \\ s, t \in S}} \abs{X_t - X_s} > 2^{-m} \right),
  \end{align*}
  where we use the fact that $\max$ of a finite number of continuous functions
  is continuous.
  But $\P \left( \max_i \max_{\substack{s, t \in T_i^m \\ s, t \in S}} \abs{X_t - X_s} > 2^{-m}
  \right) \uparrow \P \left( \max_i \sup_{\substack{s, t \in T_i^m \\ s, t \in T_0}} \abs{X_t - X_s} > 2^{-m} \right)$ (monotone convergence).
  Hence
  \begin{equation*}
    2^{-m} \geq \P \left( \sup_{\substack{\rho(s, t) < 2^{-m} \\ s, t \in T_0}} \abs{X_t - X_s} > 2^{-m} \right)
  \end{equation*}
  because $\rho(s, t) < 2^{-m} \implies \rho_m(s, t) < 1 \iff \rho_m(s, t) = 0$.
  That is, $s, t \in T_i^m$.
  By Borel--Cantelli, we have
  \begin{equation*}
    \sum_{m = 1}^\infty \P \left( \sup_{\substack{\rho(s, t) < 2^{-m} \\ s, t \in T_0}} \abs{X_t - X_s} >
      2^{-m} \right) \leq \sum_{m = 1}^\infty 2^{-m} = 1.
  \end{equation*}
  Thus as $n \to \infty$, $\abs{X_s - X_t} > 2^{-m}$ for some $s, t \in T_0$ such that
  $\rho(s, t) < 2^{-m}$ happens only \emph{finitely often}.
  That is, with probability 1, $\sup_{\rho(s, t) < 2^{-m}} \abs{X_s - X_t} \leq
  2^{-m}$ eventually (for large $m$).

  We can immediately extend $\{ X_t \}_{t \in T_0}$ to $\{ X_t \}_{t \in T}$ by
  continuity, and so $\{ X_t \}_{t \in T} \in \mathcal{C}_\rho(T, \mathbb{R})$.

  \paragraph{Part 3: Show that $X_n \cd X$ is the space $L_\infty(T)$.}
  This is equivalent to showing that for any Lipschitz function $f \colon L_\infty(T)
  \to [-1, 1]$ (i.e., $\abs{f(x) - f(z)} \leq \sup_{t \in T} \abs{x(t) - z(t)}$ for $x,
  z \in L_\infty(T)$),
  \begin{equation*}
    \E[f(X_n)] \to \E[f(X)] \quad \text{as } n \to \infty.
  \end{equation*}

  Idea: approximate by finer and finer partitions of $T$.

  Define $\pi_m(t) = t_{i, m}$ where $t \in T_i^m$ (our partitions) then $\rho(\pi_m(t),
  t) \leq 2^{-m}$.
  We showed that $X_{t}$ is uniformly continuous in the $\rho$-metric, so $(X \circ
  \pi_m)_t \defeq X_{\pi_m(t)} \to X_t$ as $n \to \infty$ (in $\sup$-norm).

  We know that $X_n \circ \pi_m$ can only take values $X_{n, \pi_m(t)} = X_{n, t_{i,
      m}}$ for $i = 1, \dots, k_m$.
  We then have that $X_n \circ \pi_m \cd X \circ \pi_m$ by the FIDI assumption.
  Thus
  \begin{equation*}
    \lim_{m \to \infty} \lim_{n \to \infty} \E \left[ f(X_n \circ \pi_m) \right] = \lim_{m \to \infty}
    \E[f(X \circ \pi_m)] = \E[f(X)]
  \end{equation*}
  for any bounded Lipschitz $f \colon L_\infty(T) \to \mathbb{R}$.
  Hence
  \begin{equation*}
    \abs{\E[f(X_n)] - \E[f(X)]} \leq \abs{\E[f(X_n \circ \pi_m) - f(X_n)]} +
    \abs{\E[f(X_n \circ \pi_m) - f(X \circ \pi_m)]} + \abs{\E[f(X \circ \pi_m) - f(X)]}.
  \end{equation*}
  Notice that the second term is $o(1)$ and the third term goes to 0 as $m \to \infty$.
  We need to bound the first term:
  \begin{align*}
    \abs{\E f(X_n) - \E f(X_n \circ \pi_m)}
    &\leq \norm{f}_{\mathrm{Lip}} \epsilon + \P \left( \sup_{t \in T} \abs{X_{n, t} - X_{n, \pi_m(t)}}
      \geq \epsilon \right) \\
    &\leq \norm{f}_{\mathrm{Lip}} \epsilon + \P \left( \sup_{\rho(s, t) < 2^{-m}} \abs{X_{n,
      s} - X_{n, t}} \geq \epsilon \right).
  \end{align*}
  The second term goes to $0$ as $m \to \infty$ by stochastic equicontinuity.
\end{proof}

\end{document}
