\documentclass[11pt]{article}

% This package contains many of the macros you may need.
%
% A useful environment defined in it is the example environment, which you can
% invoke via \begin{example} and \end{example}.
\usepackage{scribe}

\begin{document}

\begin{center}
  \fcolorbox{black}{white}{
    \begin{minipage}{.9\columnwidth}
      \textbf{Stats 300b: Theory of Statistics}
      \hfill
      \textbf{Winter 2017} \\
      \vspace{-.2cm}
      % Instead of Lecture 1, January 10, put the actual lecture number and
      % date of the lecture here
      \begin{center}
        {\LARGE Lecture 10 -- Feb 9} \\
      \end{center}
      \emph{Lecturer: John Duchi}
      \hfill
      \emph{Scribe: Qijia Jiang, Vivek Bagaria}  % You should put your name here if you scribe
    \end{minipage}
  }
\end{center}

\begin{center}
  \dbend ~~  \emph{\textbf{Warning:} these notes may contain factual errors}
\end{center}

\paragraph{Reading:} % Put the reading covered in the lecture here


\section{Sub-gaussianity}


\subsection{Definitions and Properties}
\begin{definition}
$X$ is a mean-zero $\sigma^{2}-$subgaussian RV if
\[
\mathbb{E}\big[\exp^{\lambda X}\big]\leq\exp\Big(\frac{\lambda^{2}\sigma^{2}}{2}\Big)\quad\forall\lambda\in\mathbb{R}
\]
\end{definition}

\noindent \textbf{Example: }Gaussian random variables: If $X\sim\mathcal{N}\big(0,\sigma^{2}\big)$,
then 
\[
\mathbb{E}\big[\exp^{\lambda X}\big]=\exp\Big(\frac{\lambda^{2}\sigma^{2}}{2}\Big)\quad\forall\lambda\in\mathbb{R}.
\]
Bounded random variables also fall into the category of sub-gaussian
random variables:

\noindent \textbf{Example: }
\label{exa:bounded}If $X\in[a,b]$, then $X$ is $\frac{(b-a)^{2}}{4}$
- subgaussian i.e, 
\[
\mathbb{E}\big[\exp^{\lambda\big(X-\mathbb{E}\big[X\big]\big)}\big]=\exp\Big(\frac{\lambda^{2}(b-a)^{2}}{8}\Big)\quad\forall\lambda\in\mathbb{R}
\]
\begin{proposition}
Let $X_{i}$'s be independent $\sigma_{i}^{2}$- subgaussian random
variables. Then $\sum_{i=1}^{n}X_{i}$ is a $\sum\sigma_{i}^{2}$-subgaussian
random variable.\end{proposition}
\begin{proof}
$\mathbb{E}\big[\exp^{\lambda\sum_{i=1}^{n}X_{i}}\big]=\Pi_{i=1}^{n}\mathbb{E}\big[\exp^{\lambda X_{i}}\big]=\exp\Big(\frac{\lambda^{2}\sum_{i=1}^{n}\sigma_{i}^{2}}{2}\Big).$
\end{proof}
Now that we've defined sub-gaussian random variables and a few simple
properties, lets use them to obtain concentration inequalities similar
to the Chernoff bounds.


\subsection{Concentration inequalities}
\begin{lemma}
If $X$ is a $\sigma^{2}$- subgaussian, then we have
\[
\max\big(\mathbb{P}\big(X-\mathbb{E}[X]\geq t\big),\mathbb{P}\big(X-\mathbb{E}[X]\leq-t\big)\big)\leq\exp-\Big\{\frac{-t^{2}}{2\sigma^{2}}\Big\}
\]
\end{lemma}
\begin{proof}
Let $\mathbb{E}[X]=0$ w.l.o.g. We prove the above result using
the techniques used to prove Chernoff bounds i.e, applying Markov
inequality on the exponentiation of the random variable:
\begin{eqnarray*}
\mathbb{P}\big(X\geq t\big) & = & \mathbb{P}\big(e^{\lambda X}\geq e^{\lambda t}\big)\\
 & \leq & \frac{\mathbb{E}\big[e^{\lambda X}\big]}{e^{\lambda t}}\\
 & = & e^{\frac{\lambda^{2}\sigma^{2}}{2}-\lambda t}\quad\forall\lambda\in\mathbb{R}^{+}.
\end{eqnarray*}
The LHS of the above equation is minimized for $\lambda=\frac{t}{\sigma^{2}}$,
and therefore, we have
\[
\mathbb{P}\big(X\geq t\big)\leq\exp-\Big\{\frac{-t^{2}}{2\sigma^{2}}\Big\}
\]
 \end{proof}
\begin{corollary}
(Hoeffding inequality) Let $X_{i}$ be independent $\sigma_{i}^{2}$
-subgaussian RVs. Then we have
\[
\mathbb{P}\big(\frac{1}{n}\sum_{i=1}^{n}X_{i}\geq t\big)\leq\exp-\Big\{\frac{-nt^{2}}{2\sum_{i=1}^{n}\sigma_{i}^{2}}\Big\}\quad t\geq0
\]
 and 
\[
\mathbb{P}\big(\frac{1}{n}\sum_{i=1}^{n}X_{i}\leq t\big)\leq\exp-\Big\{\frac{-nt^{2}}{2\sum_{i=1}^{n}\sigma_{i}^{2}}\Big\}\quad t<0
\]
\end{corollary}
This inequality is heaviliy used in proving concentration results
for bounded random variables (see Example \ref{exa:bounded}).


\section{Covering Number and Metric Entropy}

Let $\big(\Theta,d\big)$ be a metric space with distance measure
$d:\Theta\times\Theta\rightarrow\mathbb{R}$. 
\begin{definition}
For any $\epsilon>0$, $\big\{\theta_{i}\big\}_{i=1}^{N}$ is the
$\mathbf{\epsilon}$-\textbf{cover} of $\Theta$ if 
\[
\min_{i}d(\theta,\theta_{i})<\epsilon\quad\forall\theta\in\Theta.
\]
\end{definition}
\noindent This naturally leads to the definition of covering number:
\begin{definition}
For $\epsilon>0$, the \textbf{covering number} of $\Theta$ for
metric $d$ is 
\[
N(\Theta,d,\epsilon)=\inf\big\{ N:\exists\text{ an }\epsilon-\text{cover}\;\big\{\theta_{i}\big\}_{i=1}^{N}\text{ of }\Theta\big\}
\]
and $\log N(\Theta,d,\epsilon)$ is also referred as the \textbf{metric
entropy.}
\end{definition}
Covering a space is a task of covering the whole space with minimum
number of balls. Extending this idea, we define packing as
\begin{definition}
For any $\epsilon>0$ , $\big\{\theta_{i}\big\}_{i=1}^{M}$ is the
$\mathbf{\epsilon}$-\textbf{packing} of $\Theta$ if 
\[
\min_{i,j}d(\theta_{i},\theta_{j})>\epsilon.
\]

\end{definition}
\noindent Similar to covering number, we define packing number as
\begin{definition}
For $\epsilon>0$, the \textbf{packing number} of $\Theta$ with metric
$d$ is 
\[
M(\Theta,d,\epsilon)=\sup\big\{ N:\exists\text{ an }\epsilon-\text{packing }\;\big\{\theta_{i}\big\}_{i=1}^{N}\text{ of }\Theta\big\}
\]
 and $\log M(\Theta,d,\epsilon)$ is also referred as the \textbf{packing
entropy.}
\end{definition}
\noindent As one would suspect, covering and packing are related, and we indeed have
the relation:
\begin{equation}
M(2\epsilon)\leq N(\epsilon)\leq M(\epsilon).\label{eq:covering_packing}
\end{equation}

\noindent \textbf{Example: }Consider the balls in $\mathbb{R}^{d}$ with norm $||.||$, let
$\mathbb{B}=\big\{ V\in\mathbb{R}^{d}:||V||\leq1\big\}$, and $\Theta=r\mathbb{B}$
\begin{enumerate}
\item Since $\epsilon$- packing is equivalent of having ``disjoint'' balls
of radius $\epsilon/2$ we have 
\begin{eqnarray*}
M\text{Vol(\ensuremath{\epsilon}/2}) & \leq & \text{Vol}(r+\epsilon/2)\\
\Longrightarrow M & \leq & \Big(1+\frac{2r}{\epsilon}\Big)^{d}
\end{eqnarray*}
 
\item Similarly $\epsilon$ - covering covers the whole space with balls
of radius $\epsilon$ and hence we have
\begin{eqnarray*}
N\text{Vol}(\epsilon) & \geq & \text{Vol}(r)\\
\Longrightarrow N & \geq & \Big(\frac{r}{\epsilon}\Big)^{d}
\end{eqnarray*}
 coupling the above inequality with equation \ref{eq:covering_packing},
we obtain
\[
\Big(\frac{r}{\epsilon}\Big)^{d}\leq N(\epsilon)\leq\Big(1+\frac{2r}{\epsilon}\Big)^{d}
\]

\end{enumerate}

\section{Bracketing number}

When the underlying space $\Theta$ is a space of functions $\mathcal{F}=\big\{ f:\mathcal{X}\rightarrow\mathbb{R}\big\}$,
we can define bracketing numbers along the lines of covering ,
packing numbers. Formally,
\begin{definition}
Let $\mathcal{F}\subseteq\big\{ f:\mathcal{X}\rightarrow\mathbb{R}\big\}$
be a collection of fns with measure $\mu$. A set $\big\{[l_{i},u_{i}]\big\}_{i=1}^{N}$
of functions $\mu_{i},l_{i}:\mathcal{X}\rightarrow\mathbb{R}$ is
a $\epsilon$ - bracketing set of $\mathcal{F}$ if 
\[
\forall f\in\mathcal{F}\;\exists i\;s.t\;l_{i}\leq f(x)\leq\mu_{i}
\]
and $\int\big(\mu_{i}(x)-l_{i}(x)\big)^{p}d\mu(x)\leq\epsilon^{p}.$ 
\end{definition}
\noindent In the spirit of defining ``numbers'' for each notion of covering
we define
\begin{definition}
Bracketing number of $\mathcal{F}$ is
\[
N_{[]}\big(\mathcal{F},L_{p}(\mu),\epsilon\big)\;:=\inf\big\{ N:\exists\;\text{ a set }\big\{[l_{i},u_{i}]\big\}_{i=1}^{N}\text{ which is }\epsilon-\text{bracketing of }\mathcal{F}\big\}
\]
 \end{definition}
\begin{claim}
Let $\mathcal{F}=\big\{ m_{\theta}:\theta\in\Theta\big\}$ where $m_{\theta}$
are $L$-Lipschitz in $\theta$, theb $N_{[]}\big(\mathcal{F},L_{p},\epsilon L\big)\leq N(\Theta,||.||,\epsilon/2)$.\end{claim}
\begin{proof}
Let $\big\{\theta_{i}\big\}_{i=1}^{N}$ be an $\epsilon/2$ -covering
of $\Theta$, then lets define 
\begin{eqnarray*}
u_{i}(x): & = & m_{\theta_{i}}(x)+\frac{\epsilon}{2}L\\
l_{i}(x): & = & m_{\theta_{i}}(x)-\frac{\epsilon}{2}L.
\end{eqnarray*}
 We know that for any $\theta\in\Theta$, $\exists\theta_{i}$ s,t
$||\theta-\theta_{i}||\leq\frac{\epsilon}{2}$, and from Lipschitz
properties of $m_{\theta}$, we have
\begin{eqnarray*}
\big|m_{\theta}(x)-m_{\theta_{i}}(x)\big| & \leq & L||\theta-\theta_{i}||\\
 & \leq & \frac{\epsilon}{2}L.
\end{eqnarray*}
 \end{proof}
\begin{theorem}
(Uniform Convergence) Let $\mathcal{F}$ satisfy $N_{[]}\big(\mathcal{F},L_{p},\epsilon\big)<\infty,$
then under i.i.d. sampling 
\[
\sup_{f\in\mathcal{F}}\big|P_{n}f-Pf\big|\overset{p}{\rightarrow}0.
\]
\end{theorem}
\begin{proof}
For any given $\epsilon>0$ let $\big\{[l_{i},u_{i}\big\}_{i=1}^{N}$
be $\epsilon$-bracketing numbers then $\forall f\in\mathcal{F},\;\exists i$
s.t $l_{i}\leq f\leq u_{i}$, and therefore we have
\begin{eqnarray*}
P_{n}f-Pf & \leq & P_{n}u_{i}-Pl_{i}\\
 & = & P_{n}u_{i}-Pu_{i}+Pu_{i}-Pl_{i}\\
 & \leq & o_{p}(1)+\epsilon.
\end{eqnarray*}
Since $N_{[]}$ is finite and $\epsilon$ was arbitrary, we have
\begin{eqnarray*}
\sup_{f\in\mathcal{F}}\big|P_{n}f-Pf\big| & \leq & \big|N_{[]}\big|o_{p}(1)+\epsilon\\
 & \leq & 2\epsilon\rightarrow0.
\end{eqnarray*}
 \end{proof}

\end{document}
