\documentclass[11pt]{article}

% This package contains many of the macros you may need.
%
% A useful environment defined in it is the example environment, which you can
% invoke via \begin{example} and \end{example}.
\usepackage{scribe}

\begin{document}

\begin{center}
  \fcolorbox{black}{white}{
    \begin{minipage}{.9\columnwidth}
      \textbf{Stats 300b: Theory of Statistics}
      \hfill
      \textbf{Winter 2017} \\
      \vspace{-.2cm}
      % Instead of Lecture 1, January 10, put the actual lecture number and
      % date of the lecture here
      \begin{center}
        {\LARGE Lecture 1 -- January 10} \\
      \end{center}
      \emph{Lecturer: John Duchi}
      \hfill
      \emph{Scribe: Xiaowei Wang}  % You should put your name here if you scribe
    \end{minipage}
  }
\end{center}

\begin{center}
  \dbend ~~  \emph{\textbf{Warning:} these notes may contain factual errors}
\end{center}

\paragraph{Reading:} VDV Chapter 2.1, 2.2
% Put the reading covered in the lecture here


%%%%%%%%%%%%%%%%% SCRIBE NOTES HERE %%%%%%%%%%%%%%%%%%%%

\paragraph{Outline of lecture 1:}
\begin{itemize}
\item Administrative basic stuff: As on the syllabus
\item Overview of the course
\item Basic theory of convergence of random variables
\item Probability, Asymptotic Statistics and Distributions
\end{itemize}

\paragraph{Overview of the course:} 
(In this course, we will be majorly dealing with big data sets, $N\rightarrow\infty$)

1. Convergence of random variables, random vectors, estimators and functions.

2. Understanding various notions of optimality and quality of estimators and tests. We will not be talking about admissibility as it is too difficult. What we will try to do in this course is to show that certain estimators are good under specific metrics or to prove that certain estimators are unimprovable.

\subparagraph{Part I of the course:} Finite dimensional problems and statistic models

\begin{example}
	One example problem is that we have $X_i \thickspace \simiid P_\theta, X_i\in \reals^d$, where $d$ is fixed. We want to understand the estimators of parameter $\theta\in\reals^d$ of distribution $P_\theta$.
\end{example}

\subparagraph{Part II of the course:} Optimality and comparisons of estimators

In this part, we will try to understand when an estimator $\hat{\theta}$ of $\theta$ is good or optimal. Also, we will look into how to distinguish $P_\theta$ from $P_{\theta+\Delta}$ when $\Delta$ is small.

\subparagraph{Part III of the course:} Infinite dimensional or uniform laws of convergence for random variables

1. Concentration inequalities

2. For functions $F: X\times \scriptsize{\theta} \rightarrow \reals$, we will look into how 
$$\dfrac{1}{n}\sum\limits_{i=1}^nF(x_i,\theta)\rightarrow \E[F(x,\theta)]$$
uniformly in $\theta$.

\paragraph{Backgrounds needed:}

1. Stat 300a (not strictly necessary).

2. Probability at stat 310a level. 

e.g. Convergence of distribution, Helly Selection Theorem etc.

3. Analysis at Math 171 level. 

e.g. Compactness, metric spaces etc.



\paragraph{Basic theory of convergence of random variables:} \mbox{}\\

In this part we will go thourgh basic definitions, Continuous Mapping Theorem and Portmanteau Lemma.

For now, assume $X_i\in \reals^d, d<\infty$. We first give the definition of various convergence of random variables.

\begin{definition}(Convergence in probability)
We call $X_n\cp X$ (sequence of random variables converges to X) if
$$\lim\limits_{n\rightarrow \infty}\P(||X_n-X||\geq \epsilon)=0,\thickspace\forall \epsilon>0$$	
\end{definition}

In a general metric space, with metrix $\rho$, the above definition becomes
$$\lim\limits_{n\rightarrow \infty}\P(\rho(X_n,X)\geq \epsilon)=0,\thickspace\forall \epsilon>0$$

\begin{definition}(Weak convergence or convergence in distribution)
	
    We say $$X_n\cd X $$
	if for $\forall x\in \reals^d$,
	$$\P(X_n\leq x)\rightarrow \P(X\leq x)$$
	at all $X\in \reals^d$ such that $x\rightarrow \P(X\leq x)$ is continuous. 
\end{definition}

Note: In the above definition $\P(X\leq x)=\P(X\in(-\infty,x_1]\times\cdots\times(-\infty,x_d])$ 

We also have an alternative definition for convergence in distribution.

\begin{definition}
	$$X_n\cd X $$
	if for all bounded continuous function $f$, $$\E[f(X_n)]\rightarrow \E[f(X)]$$
\end{definition}

Below is a definition of $L^p$ convergence.

\begin{definition}(Convergence in the $p^{th}$ mean)
	
	We say that $$X_n\overset{L^p}{\rightarrow} X $$
	if $$\lim\limits_{n\rightarrow \infty} \E[||X_n-X||^p]=0$$
\end{definition}

Finally, we give the definition of almost surely convergence for random variables.

\begin{definition}($X_n$ converges almost surely to $X$)
	
	We say that $$X_n\cas X $$
	if $$\P(\lim\limits_{n\rightarrow \infty}X_n\neq X)=0$$
	i.e.
	$$\P(\lim\limits_{n\rightarrow \infty}||X_n-X||\geq\epsilon)=0, \thickspace\forall \epsilon>0$$
\end{definition}

\paragraph{Standard implications:} \mbox{}\\

For the various types of convergence above, we have the following relationships.

$$X_n\cas X \Rightarrow X_n\cp X \Rightarrow X_n\cd X$$
$$X_n\overset{L^p}{\rightarrow} X \Rightarrow X_n\cp X$$

All the reversed directions may not be true.

Examples of almost surely convergence and convergence in probability can be found in the strong law of large numbers and central limits theorem, as stated below.

\begin{example}
	If $X_i \simiid P$, $cov(X_i)=\Sigma=\E[(X_i-\mu)(X_i-\mu)^T]$, $\mu=\E[X_i]$, then
	$$\dfrac{1}{n}\sum\limits_{i=1}^nX_i\cas\mu$$
	$$\dfrac{1}{\sqrt{n}}\sum\limits_{i=1}^nX_i\cd \normal(0,\Sigma)$$	
\end{example}

\subparagraph{Basic Convergence Theorems:}(Chapter VDV for all proofs)

\begin{theorem}(Continuous Mapping Theorem)
	Let $g$ be continuous on a set $B$ such that $\P(X\in B)=1$
	then
	$$X_n\cp X \Rightarrow g(X_n)\cp g(X)$$
	$$X_n\cas X \Rightarrow g(X_n)\cas g(X)$$
	$$X_n\cd X \Rightarrow g(X_n)\cd g(X)$$
\end{theorem}

For the heuristics of the third line: If $g$ is continuous, then $f\circ g$ is continuous and bounded for any continuous bounded $f$. Thus,
$$\E[f(g(X_n))]\rightarrow \E[f(g(x))]$$

Another important theorem we will need is Slutsky's Theorem.

\begin{theorem}(Slutsky's Theorem)
	
	(1) If $c$ is constant, then
	$$X_n\cd c\iff X_n\cp c$$
	
	(2) If $X_n\cd X$, $d(X_n,Y_n)\cp0$, then
	$$Y_n\cd X$$
	
	(3) If $X_n\cd X$, $Y_n\cp c$, then
	$$\left( \begin{array}{c} X_n \\ Y_n \end{array} \right)\cd \left( \begin{array}{c} X \\ c \end{array} \right)$$
\end{theorem}

The Slutsky's theorem allows us to ignore low order terms in convergence. Also, the following example shows that stronger impliations over part (3) may not be true.

\begin{example}
	If $X_n\cd \normal(0,I)$, then $-X_n\cd \normal(0,I)$. 
	
	However,
	$$\left( \begin{array}{c} X_n \\ -X_n \end{array} \right)\cd \left( \begin{array}{c} Z \\ -Z \end{array} \right)$$	where $Z\sim \normal(0,I)$ instead of $\normal(0,I)$.
\end{example}

\begin{proof-sketch}

(1) The $"\Leftarrow"$ direction is trivial and given in the previous sections. For $"\Rightarrow"$ direction of the theorem, take
$$f(x)=||x-c||\wedge 1=\min \{||x-c||,1\}$$
then $$\E[||x_n-c||\wedge 1]\rightarrow 0$$

(2) Let $f$ be 1-Lipschitz and bounded by 1, then we have
$$\E[f(Y_n)]\in \E[f(X_n)]\pm \E[d(X_n,Y_n)\wedge 1]$$
Since $\E[f(X_n)]\rightarrow \E[f(X)]$ and $\E[d(X_n,Y_n)\wedge 1] \rightarrow 0$, we have
$$\E[f(Y_n)]\rightarrow \E[f(X)]$$
and thus $Y_n\rightarrow X$.

(3) We have $$\left( \begin{array}{c} X_n \\ Y_n \end{array} \right)-\left( \begin{array}{c} X \\ c \end{array} \right)=\left( \begin{array}{c} 0\\ Y_n-c \end{array} \right) \cp 0$$
By part (2), 
$$\left( \begin{array}{c} X_n \\ c \end{array} \right)\cd \left( \begin{array}{c} X \\ c \end{array} \right)\Rightarrow \left( \begin{array}{c} X_n \\ Y_n \end{array} \right)\cd \left( \begin{array}{c} X \\ c \end{array} \right)$$

\end{proof-sketch}

\subparagraph{Consequences of Slutsky's Theorem:}\mbox{}\\

If $X_n\cd X$, $Y_n\cd c$, then
$$X_n+Y_n \cd X+c$$
$$Y_nX_n\cd cX$$
If $c\neq 0$,
$$\frac{X_n}{Y_n}\cd \frac{X}{c}$$

\begin{proof}
Apply Continuous Mapping Theorem and Slutsky's Theorem and the statements can be proved.
\end{proof}

Note: For the third line of convergence, if $c\in \reals^{d\times d}$ is a matrix, then (2) still holds. Moreover, if $\det(c)\neq0$, (3) holds but
$$Y_n^{-1}X_n\cd c^{-1}X$$
because $c\rightarrow c^{-1}$ is continuous when  $\det(c)\neq0$.

\begin{example}(t-type statistics:)
	Let $X_i\simiid P$, $\cov(X_i)=\Gamma>0$, define
	$$\mu_n=\dfrac{1}{n}\sum\limits_{i=1}^nX_i$$
	$$S_n=\dfrac{1}{n}\sum\limits_{i=1}^n(X_i-\mu_n)(X_i-\mu_n)^T$$
	$$T_n=\dfrac{1}{\sqrt{n}}S_n^{-\frac{1}{2}}\sum\limits_{i=1}^n(X_i-\mu)$$
	Then $T_n\cd \normal(0,I)$.
	
	The reason is that
	$$\mu_n\cp\E[X]$$
	and
	$$S_n\cp\Gamma$$
	Apply Slutsky's Theorem,
	$$T_n-\dfrac{1}{\sqrt{n}}\Gamma^{-\frac{1}{2}}\sum\limits_{i=1}^n(X_i-\mu)\cp 0$$
\end{example}
 
\paragraph{Big-O Notation:}(in probability)

In this part we introduce the big-o and little-o notation in probbility.

Let $X_n$ be random vectors, $R_n$ be random variables. We say that $X_n=o_p(R_n)$ if $\exists$ random vectors $Y_n$ such that 
$$X_n=Y_nR_n$$
$$Y_n\cp 0$$
This is called "little o-pea".

We say that $X_n=O_p(R_n)$ if $\exists$ random vectors $Y_n$ where $Y_n=O_p(1)$. $Y_n=O_p(1)$ means that means $\{Y_n\}$ is uniformly tight. i.e.
$$\limsup\limits_{M\rightarrow \infty}  \sup\limits_{n}\P(||Y_n||\geq M)=0$$
or $\forall \epsilon>0$, $\exists$ $M$ such that 
$$\P(||Y_n||\geq M)\leq \epsilon, \forall \thickspace n$$

\subparagraph{Comsequences:}\mbox{\\}

With the definition above, we can get the following properties and lemma.

$$o_p(1)+o_p(1)=o_p(1)$$
$$O_p(1)+o_p(1)=O_p(1)$$
$$O_p(1)+O_p(1)=O_p(1)$$

\begin{lemma}
	Let function $R: \reals^d\rightarrow \reals^k$, with $R(0)=0$. Let $X_n\cp 0$, then
	
	(1) If $R(h)=o(||h||^p)$ as $h\rightarrow 0$, then
	$$R(X_n)=o_p(||X_n||^p)$$
	
	(2) If $R(h)=O(||h||^p)$ as $h\rightarrow 0$, then
	$$R(X_n)=O_p(||X_n||^p)$$
\end{lemma}

\begin{proof}
	Define
	$$g(h)=\left\{\begin{aligned}\dfrac{R(h)}{||h||^p} &, if\thickspace h\neq 0\\0&, if\thickspace h=0\end{aligned}\right.$$
	
	(1) Then $g(h)\rightarrow 0$ as $h\rightarrow 0$. Thus, $g$ is continuous at 0 and $X_n\cp 0$. Apply Continuous Mapping Theorem(CMT), we get
	$$g(X_n)\cp 0$$
	
	(2) $\exists$ $M$, $\delta>0$ such that $||g(h)||\leq M$, $\forall ||h||\leq \delta$.
	Then $$\Phi(||g(X_n)||>M)\leq \P(||X_n||\geq \delta)\rightarrow 0$$
	so $$g(X_n)=O_p(1)$$

	
\end{proof}



\end{document}
