\documentclass[11pt]{article}

% This package contains many of the macros you may need.
%
% A useful environment defined in it is the example environment, which you can
% invoke via \begin{example} and \end{example}.
\usepackage{scribe}
\usepackage{bbm}
\newcommand{\cc}{\mathcal{C}}
\newcommand{\X}{\mathcal{X}}
\newcommand{\F}{\mathcal{F}}
\begin{document}

\begin{center}
  \fcolorbox{black}{white}{
    \begin{minipage}{.9\columnwidth}
      \textbf{Stats 300b: Theory of Statistics}
      \hfill
      \textbf{Winter 2017} \\
      \vspace{-.2cm}
      % Instead of Lecture 1, January 10, put the actual lecture number and
      % date of the lecture here
      \begin{center}
        {\LARGE Lecture 13 -- February 21} \\
      \end{center}
      \emph{Lecturer: John Duchi}
      \hfill
      \emph{Scribe: Zhimei Ren}  % You should put your name here if you scribe
    \end{minipage}
  }
\end{center}

\begin{center}
  \dbend ~~  \emph{\textbf{Warning:} these notes may contain factual errors}
\end{center}

\paragraph{Reading:} % Put the reading covered in the lecture here

\paragraph{Outline}
\begin{itemize}
\item VC classes
\item General convergence in distribution:
\begin{itemize}
\item in metric space
\item uniform laws in function spaces
\item compactness in function spaces
\end{itemize}
\end{itemize}

\section{Recap}
\begin{definition}
{\bf (Vapnik-Chervonenkis classes)} $\mathcal{C}$(= collection of sets) shatters $x_1,...,x_n$ if for all labelings $y\in\{\pm1\}^n$ of $\{x_i\}$ $\exists C\in\mathcal{C}$, s.t. 
\[\begin{cases}
y_i=1&x_i\in C\\
y_i=0&x_i\notin C 
\end{cases}\]
VC($\cc$)=size of largest set $x_1,...,x_n$ shattered by $\cc$.
\end{definition}

\section{VC Classes}
\begin{theorem}
{\bf(Uniform covering numbers in $L_r(P)$)}For sets A,B, let dist(A,B)=$||\mathbbm{1}_{A}-\mathbbm{1}_{B}||_{L_r(P)}=(\int|\mathbbm{1}_{A}-\mathbbm{1}_{B}|^rdP)^{\frac{1}{r}}$. $\exists$ constant $K<\infty$,
$$
\sup_P N(\cc,Lr(P),\epsilon)\leq K VC(\cc)(4e)^{VC(\cc)}(\dfrac{1}{\epsilon})^{rVC(\cc)}
$$
i.e. 
$$
\log N(\cc,Lr(P),\epsilon)\lesssim rVC(\cc)\log(\dfrac{1}{\epsilon})
$$
\end{theorem}

\begin{example}
Let $\mathcal{F}=\{f(x)=\mathbbm{1}_{X\leq t},t\in\mathbb{R}^d\}$. vc($\mathcal{F}$)=$O(d)$.
$$
\sup_P\log N(\mathcal{F},L_2(P),\epsilon)\leq Kd\log(\dfrac{1}{\epsilon})
$$
As a consequence, we have the classical Glivenko Cantelli theorem:

\begin{equation*}
\begin{aligned}
\E [\sup_{t\in \mathbb{R}^d}|\P_n(X\leq t)-\P (X\leq t)|]&=\E[\sup_{f\in\mathcal{F}}|\P_n f-\P f|]\\
&\leq \dfrac{2}{\sqrt{n}}\E[|\sup_{f\in\mathcal{F}|}\dfrac{1}{\sqrt{n}}\sum^n_{i=1}\epsilon_if(X_i)](\epsilon_i\overset{i.i.d}{\sim}\{\pm 1\})\\
&\overset{Dudley}{\leq}\dfrac{const}{\sqrt{n}}\int^1_0\sqrt{\log N(\mathcal{F},L_2(P_n),\epsilon)}d\epsilon\\
&=\dfrac{const\sqrt{d}}{\sqrt{n}}\int^1_0\sqrt{\log(1/\epsilon)}d\epsilon\\
&\leq \dfrac{\tilde{const}\sqrt{d}}{\sqrt{n}}
\end{aligned}
\end{equation*}
\end{example}

\begin{definition}
The subgraph of a function: $\mathcal{X}\rightarrow \mathbb{R}$:
$$
\textrm{sub}f:=\{(x,t):t<f(x)\}=(\textrm{epi}f)^c
$$
\end{definition}
Note: sub$f\subseteq\mathcal{X}\subseteq\R$.

\begin{definition}
$\mathcal{F}$ is a VC-class (VC-subgraph-class) if $\textrm{sub}f:f\in\mathcal{F}$ is VC.
\end{definition}

\begin{example}
Let $\mathcal{F}=\{f=<\theta,x>:\theta\in \R^d\}$, then VC$(\mathcal{F})\leq d+2$
\end{example}

\begin{theorem}
If VC($\mathcal{F})<\infty$ and $\mathcal{F}$ has envelope $F:\X\rightarrow \R_+$(i.e.$F(x)\geq |f(X)|$, all $f \in \F$).
$$
\sup_P N(\F,L_r(P),||F||_{L_r(P)}\epsilon)\leq const VC(\F) (16e)^{VC(\F)}(\dfrac{1}{\epsilon})^{rVC(\F)}
$$
\end{theorem}

\begin{example}
Classification problem. Given $(X_i,y_i)\overset{i.i.d}{\sim}P$,$X_i\in \R^d,y_i\in\{\pm 1\}$, choose vector $\theta\in \R^d$ to make $\dfrac{1}{n}\sum^n_{i=1}\mathbbm{1}_{sign(\theta^Tx)\neq y_i}$ small. i.e. we learn/ fit classifier with rule 
$$
\hat{y}|x=sign(\theta^Tx)
$$

$\F=\{<\theta,x>,\theta\in\R^d\}$
\vskip 1em
Therefore
$$
\E[\sup_{\theta\in\Theta}|\P_n(sign(\theta^Tx)\neq y_i)-\P(sign(\theta^Tx)\neq y_i)|]\leq const\sqrt{\dfrac{d}{n}} (\textrm{by entropy integral})
$$
If you have theta with good empirical error, you should have low error under $\P$.
\end{example}

{\bf Properties of VC classes convenient for analysis:}
\begin{enumerate}
\item if $\F$ is a linear space of functions, dim($\F<\infty$), then VC($\F$)=$O(dim(\F))$.
\item If $\cc$ and $\mathcal{D}$ are VC classes of sets,
\[
\cc \cup \mathcal{D}:=\{C\cup D, C\in\cc, D\in \mathcal{D} \}
\]
\[
\cc \cap \mathcal{D}:=\{C\cap D, C\in\cc, D\in \mathcal{D} \}
\]
are VC.
\item Comparison: if $\F$ is VC classof functions, then if $\phi:\R\rightarrow\R$ is monotone 
$\phi\circ f:f\in\F$ is VC.
\end{enumerate}

\section{Convergence in distribution in metric spaces and uniform CLTs}

Our goal here is that given collections of functions $\F$, when is there a limiting Gaussian for $[\sqrt{n}(\P_n f-\P f)]_{f\in\F}$ in a uniform sense? Our starting point is Weak convergence/convergence in distribution.

\vskip 1em

Recall $X_n\overset{d}{\rightarrow}X$ is equivalent to either of the following:
\begin{enumerate}
\item $\E[f(X_n)]\rightarrow\E[f(X)]$ for all bounded and continuous $f$ 
\item  $\E[f(X_n)]\rightarrow\E[f(X)]$ for all Lipschitz and bounded function $f$.
\end{enumerate}


1 and 2 makes sense even if $X_n$ are in some metric space.

\vskip 1em

Let $\mathbb{D}$ be a metric space. Then X is a r.v. on $\mathbb{D}$ if $X:\Omega\rightarrow\mathbb{D}$. Say X is $\mathbb{D}$-valued. Then given sequence of $X_n:\Omega_n\rightarrow \mathbb{D}$. We say $X_n\rightarrow X$ if $\E[f(X_n)]\rightarrow\E[f(X)]$ for all bounded and continuous $f$ (even Lipschitz). (Recall $\E f(X_n)=\int_{\Omega_n}f(X_n(\omega))d\mu_n(\omega))=\int f(x)\P_n(x)$)

\vskip 1em

\begin{example}
(Continuous function in a compact set) Let $(T,d)$ be a compact metric space. $L_{\infty}(T)$=bounded functions: $T\rightarrow \R$. For $f,g\in L_{\infty}(T)$, $||f-g||_{\infty}=\sup_{t\in T}|f(t)-g(t)|$. Let $l: T\times \X\rightarrow \R$ be continuous in t. Define
$$
Z_n(.)=\dfrac{1}{\sqrt{n}}\sum^n_{i=1}[l(.,x)-\E l(.,x)]
$$
Then $Z_n$ is a $l_{\infty}$-valued random variable. (Because $t\rightarrow Z_n(t)$ is contiunous, so $\sup_{t\in T}|Z_n(t)|<\infty$)

\vskip 1em
\begin{remark}
Note if $T_0$ is countable and dense subset of $T$, $Z_n$ is completely determined by $\{Z_n(t),t\in T_0\}$.
$$
(Z_n(t_i),...,Z_n(t_k))\overset{d}{\rightarrow}\mathcal{N}(0,\textrm{cov}(l(t_i,x),l(t_j,x))^k_{i,j=1})
$$
for fixed $t_i,...,t_k$.
\end{remark}
\end{example}

\begin{definition}
A random variable $X:\Omega\rightarrow \mathbb{D}$ is tight if $\forall \epsilon>0$, $\exists$ a compact set $K \subseteq \mathbb{D}$

$$
\limsup_{n\rightarrow\infty}\P(X_n\notin K^{\delta})\leq \epsilon
$$
all $\delta>0$.

$K^{\delta}:=\{x:dist(x,K)<\delta\}$
\end{definition}

\begin{theorem}
(Prohorov) Let $X_n:\Omega\rightarrow \mathbb{D}$ and $X:\Omega\rightarrow \mathbb{D}$
\begin{enumerate}
\item If $X_n\overset{d}{\rightarrow} X$, where $X$ is tight, then $\{X_n\}$ is asymptotically tight.
\item If $\{X_n\}$ is asymptotically tight, then $\exists$ a subsequence $\{n_k\}$, tight $X:\Omega\rightarrow \mathbb{D},s.t.$ $X_{n_k}\overset{d}{\rightarrow}X$.
\end{enumerate}
\end{theorem}

{\bf Compactness in functional space:}
\vskip 1em

Evidently, if $X_n\in L_{\infty}(T)$, we must understand compactness in $L_{\infty}(T)$. For us, limits will be in $\cc(T,R)$. $\cc(T,R)=$continuous function. $f: T\rightarrow \R$, $||f-g||_{\infty}$ is metric.
\vskip 1em

{\bf Standard compactness theorem: Arzela-Ascoli theorem}
\begin{definition}
For a function $f:T\rightarrow \R$, modulus of continuity is
$$
w_f(\delta)=\sup_{d(s,t)<\delta}\{|f(t)-f(s)|\}
$$
\end{definition}


\begin{definition}
A collection $\F$ is uniformly equicontinuous if 
$$
\lim_{s\downarrow 0} \sup_{f\in\F} w_f(\delta)=0
$$
\end{definition}

\begin{theorem}
Let $(T,d)$ be a compact metric space. Then:
\begin{enumerate}
\item  $\F\subset \cc(T,\R)$ is sequentially compact.
\item $\F$ is uniformly equicontinuous and $\exists t_0\in T,$ s.t. $\sup_{f\in\F}|f(t_0)|<\infty$
\end{enumerate}
are equivalent.
\end{theorem}



%%%%%%%%%%%%%%%%% SCRIBE NOTES HERE %%%%%%%%%%%%%%%%%%%%


\end{document}