\documentclass[11pt]{article}

% This package contains many of the macros you may need.
%
% A useful environment defined in it is the example environment, which you can
% invoke via \begin{example} and \end{example}.
\usepackage{scribe}

\begin{document}

\begin{center}
  \fcolorbox{black}{white}{
    \begin{minipage}{.9\columnwidth}
      \textbf{Stats 300b: Theory of Statistics}
      \hfill
      \textbf{Winter 2017} \\
      \vspace{-.2cm}
      % Instead of Lecture 1, January 10, put the actual lecture number and
      % date of the lecture here
      \begin{center}
        {\LARGE Lecture 6-- January 26} \\
      \end{center}
      \emph{Lecturer: John Duchi}
      \hfill
      \emph{Scribe: Yan Chen, Xiaoyan Han }  % You should put your name here if you scribe
    \end{minipage}
  }
\end{center}

\begin{center}
  \dbend ~~  \emph{\textbf{Warning:} these notes may contain factual errors}
\end{center}

\paragraph{Reading:} VDV Chapter 12

\paragraph{Outline:}

\begin{itemize}
\item Efficiency of Estimators
\begin{itemize}
\item Super Efficiency
\end{itemize}
\item U-Statistics (VDV Chapter 12)
\begin{itemize}
\item Definitions
\item Examples
\item Variance
\end{itemize}
\end{itemize}

\section{Efficiency of Estimators}

\paragraph{Recall:} If  $\hat{\theta}_{n}$ and $T_{n}$   are estimators/statistics such that $\sqrt{n}\left(\hat{\theta}_{n}-\theta\right)\overset{d}{\to}N\left(0,\sigma^{2}\right)$
for some $m\left(n\right)\to\infty$, and $\sqrt{n}\left(T_{m\left(n\right)}-\theta\right)\overset{d}{\to}N\left(0,\sigma^{2}\right)$,
then the ARE (Asymptotic Relative Efficiency) of $\hat{\theta}_{n}$
w.r.t. $T_{n}$ is $\lim\inf_{n\to\infty}\frac{m\left(n\right)}{n}$. 

\begin{observation}{ (Pitman Efficiency): }
Say that $\sqrt{n}\left(\hat{\theta}_{n}-\theta\right)\overset{d}{\to}N\left(0,\sigma^{2}\left(\theta\right)\right)$
and $\sqrt{n}\left(T_{n}-\theta\right)\overset{d}{\to}N\left(0,\tau^{2}\left(\theta\right)\right).$
Then the ARE of $\hat{\theta}_{n}$ w.r.t. $T_{n}$ is $\frac{\tau^{2}\left(\theta\right)}{\sigma^{2}\left(\theta\right)}$
in the one-dimensional case. (In higher dimensions, it is roughly
$\text{tr}\left(\tau^{2}\left(\theta\right)\left(\sigma^{2}\left(\theta\right)\right)^{-1}\right)$). 
\end{observation}

\begin{proof}
Let $m\left(n\right)=\left\lceil \frac{\tau^{2}}{\sigma^{2}}n\right\rceil $.
Then, 

\[
\sqrt{n}\left(T_{m\left(n\right)}-\theta\right)=\underset{\to\frac{\sigma}{\tau}}{\underbrace{\sqrt{\frac{n}{m\left(n\right)}}}}\underset{\overset{d}{\to}N\left(0,\tau^{2}\right)}{\underbrace{\sqrt{m\left(n\right)}\left(T_{m\left(n\right)}-\theta\right)}}\underset{\text{Slutsky's }}{\underbrace{\overset{d}{\to}}}N\left(0,\sigma^{2}\left(\theta\right)\right)
\]

Roughly, if $\tau^{2}>\sigma^{2}$ , we prefer $\hat{\theta}_{n}$  to $T_{n}$  because $T_{n}$  requires $\frac{\tau^{2}}{\sigma^{2}}$ times the sample size that $\hat{\theta}_{n}$  does for a similar quality.
\end{proof}

\subsection{Super-Efficiency}

\begin{definition}[Comparison of Estimators and Super-Efficiency]

Recall if $I(\theta)$  is Fisher-Information for the model $\{P_{\theta}\}_{\theta\in\Theta}$   and if $\sqrt{n}\left(\hat{\theta}_{n}-\theta\right)\underset{P_{\theta}}{\overset{d}{\to}}N\left(0,I\left(\theta\right)^{-1}\right)$
, then $\hat{\theta}_{n}$ is efficient. If $\sqrt{n}\left(\hat{\theta}_{n}-\theta\right)\underset{P_{\theta}}{\overset{d}{\to}}N\left(0,\sigma^{2}\left(\theta\right)\right)$,
where $\sigma^{2}\left(\theta\right)\leq I\left(\theta\right)^{-1}$
and there is some $\theta_{0}\text{ such that }\sigma^{-1}\left(\theta_{0}\right)<I\left(\theta\right)^{-1}$,
then we call $\hat{\theta}_{n}$ super-efficient.

\end{definition}

\begin{example*}
(Hodges Estimator for Gaussian Mean)

For $X_{i}\overset{i.i.d}{\sim}N\left(\theta,1\right)$ and $\hat{\theta}_{n}=\bar{X}_{n}=\frac{1}{n}\sum_{i=1}^{n}X_{i}$,
let $T:=\begin{cases}
\bar{X}_{n} & \text{if }\left|\bar{X}_{n}\right|\geq n^{-1/4}\\
0 & \text{if }\left|\bar{X}_{n}\right|<n^{-1/4}
\end{cases}.$

If $\theta=0$, then

\[
P_{\theta}\left(\sqrt{n}T_{n}=0\right)=P_{\theta}\left(\left|\bar{X}_{n}\right|\leq n^{-1/4}\right)=P_{\theta}\left(\left|\underset{N\left(0,1\right)}{\underbrace{\sqrt{n}\bar{X}_{n}}}\right|\leq n^{1/4}\right)\to1\ \text{as}\ n\to\infty.
\]

Therefore, $\sqrt{n}\left(T_{n}\right)\overset{d}{\underset{P_{0}}{\to}}0$. 

If $\theta\neq0$, then

\[
\sqrt{n}\left(T_{n}-\theta\right)=\underset{\overset{d}{\to}N\left(0,1\right)}{\underbrace{\sqrt{n}\left(\bar{X}_{n}-\theta\right)}}\underset{\overset{a.s}{\to}1}{\underbrace{\mathbf{1}\left\{ \left|\bar{X}_{n}\right|\geq n^{-1/4}\right\} }}+\sqrt{n}\left(0-\theta\right)\underset{\overset{a.s}{\to}0}{\underbrace{\mathbf{1}\left\{ \left|\bar{X}_{n}\right|<n^{-1/4}\right\} }}\overset{d}{\to}N\left(0,1\right)
\]

So $\sqrt{n}\left(T_{n}-\theta\right)\overset{d}{\to}\begin{cases}
0 & \text{if }\theta=0\\
N\left(0,1\right) & \text{o.w. }
\end{cases}.$

This is a bad estimator. We will explore this in the homework.
\\
\end{example*}

\section{U-Statistics}

\subsection{Definitions}

\par
Suppose I have $h:\ X^{r}\to\mathbb{R}$ and want to estimate $\theta=E\left[h\left(X_{1},...,X_{r}\right)\right]$
where the $X_{i}$ are independent. Given a sample $X_{1},...,X_{n}$,
how should I estimate $\theta$? (Why care about this?)


\begin{example*}

Observe that 

\[
\text{Var}\left(X\right)=E\left[X_{1}^{2}\right]-E\left[X_{1}X_{2}\right]=\frac{1}{2}E\left[\left(X_{1}-X_{2}\right)^{2}\right].
\]

So,

\[
h\left(X_{1},X_{2}\right)=\frac{1}{2}\left(X_{1}-X_{2}\right)^{2}
\]
\\
\end{example*}

\begin{remark}
Always, without loss of generality, we assume $h$ is symmetric.
\end{remark}

\begin{paragraph}{}

I should estimate $\theta$  with with U-Statistics (Hoeffding 1940s).  It allows us to (1) abstract away annoying details and still perform inference and (2) develop statistics and tests that do not depend on parametric assumptions (non-parametric) while things are parametric.

\end{paragraph}

\begin{definition}[U-Statistics]

Let $h:\ X^{r}\to\mathbb{R}$ be symmetric. (Kernel Function). For
$X_{i}\overset{i.i.d}{\sim}P$, define $\theta\left(P\right):=E_{P}\left[h\left(X_{1},...,X_{r}\right)\right]$
and

\[
U_{n}:=\frac{1}{\choose{n}{r}} \sum_{\left|\beta\right|=r,\beta\subset\left[n\right]}h\left(X_{\beta}\right)
\]

where $\beta$ ranges over size $r$ subsets of $\left[n\right]=\left\{ 1,...,n\right\} $,
$X_{\beta}=\left(X_{i_{1}},...,X_{i_{r}}\right)$ for $\beta=\left(i_{1},...,i_{r}\right)$.

\end{definition}

\begin{remark}
$E_{P}\left[U_{n}\right]=\theta\left(P\right),$ so it is unbiased.
\end{remark}

Why use an U-statistic at all? Why not

\[
\frac{1}{\left(\frac{n}{r}\right)}\sum_{\ell=1}^{\frac{n}{r}}h\left(X_{\ell\left(r-1\right)+1},...,X_{\ell r}\right)?
\]

Let $\left\{ X_{(1)},...,X_{(n)}\right\} $ be the sample with ``index''
information removed. (e.g. Order Statistics. Generally a histogram.
For EE, called ``type'' of the sample.) Then, under $X_{i}\overset{i.i.d}{\sim}P$,
then $\left\{ X_{(i)}\right\} _{i=1}^{n}$ is a sufficient statistic
for everything. Observe that

\[
E\left\{ h\left(X_{1},...,X_{r}\right)|X_{(1)},...,X_{(n)}\right\} =U_{n}:=\frac{1}{{ \choose {n}{r}}}\sum_{\left|\beta\right|=r,\beta\subset\left[n\right]}h\left(X_{\beta}\right)
\]

By Rao-Blackwellization,

\[
\text{Var}\left(U_{n}\right)\leq\text{Var}\left(\underset{\text{of form }\text{Avg}\left(h\left(X_{\beta}\right),\ \left|\beta\right|=r\right)}{\underbrace{\text{Other Unbiased Estimators}}}\right).
\]

\subsection{Examples}

\begin{example*}[Sample Variance]
Consider $h\left(X_{1},X_{2}\right)=\frac{1}{2}\left(X_{1}-X_{2}\right)^{2}$
and $E\left[h\left(X_{1},X_{2}\right)\right]=\frac{1}{2}\left(E\left[X_{1}^{2}\right]+E\left[X_{2}^{2}\right]\right)-E\left[X_{1},X_{2}\right]=\text{Var}\left(X\right)$

\begin{align*}
U_{n} & =\frac{1}{\choose{n}{2}}\sum_{1\le i<j\leq n}\frac{1}{2}\left(X_{i}-X_{j}\right)^{2}\\
 & =\frac{1}{2n\left(n-1\right)}\sum_{i,j}\left(X_{i}-X_{j}\right)^{2}\\
 & =\frac{1}{2n\left(n-1\right)}\sum_{i,j}\left(\left(X_{i}-\bar{X}_{n}\right)-\left(X_{j}-\bar{X}_{n}\right)\right)^{2}\\
 & =\frac{1}{2n\left(n-1\right)}\sum_{i,j}\left(\left(X_{i}-\bar{X}_{n}\right)^{2}+\left(X_{j}-\bar{X}_{n}\right)^{2}\right)\\
 & =\frac{1}{n-1}\sum_{i=1}^{n}\left(X_{i}-\bar{X}_{n}\right)^{2}
\end{align*}
\end{example*}

\begin{example*}[Gini's Mean-Difference]

\[
h\left(X_{1},X_{2}\right)=\left|X_{1}-X_{2}\right|\text{ and }E\left[U_{n}\right]=E\left[\left|X_{1}-X_{2}\right|\right]
\]
\end{example*}

\begin{example*}[Quantiles]

\[ \theta\left(P\right)=P\left(X\leq t\right)=\int_{-\infty}^{t}dp\text{ and }h\left(X\right)=\mathbf{1}\left\{ X\leq t\right\} \]

This is a 1st order U-statistic.
\\
\end{example*}

\begin{example*}[Signed Rank Statistic]
Suppose we want to know if central location of $P$ is 0 or not. (Even
if $E\left[X\right]$ does not exist.)

\[
\theta\left(P\right):=P\left(X_{1}+X_{2}>0\right)
\]
\\
\begin{remark}
If $X_{1},X_{2}$ are symmetric about $0$ and $\theta\left(P\right)=\frac{1}{2}$,
then $h\left(X_{1},X_{2}\right)=\mathbf{1}\left\{ X_{1}+X_{2}>0\right\} $
and $U_{n}=\frac{1}{{ \choose {n}{2}}}\sum_{i<j}\mathbf{1}\left\{ X_{i}+X_{j}>0\right\} .$
\end{remark}\\
\\
\end{example*}

\begin{definition}[Two-sample U-Statistic]
For the samples $\left\{ X_{1},...,X_{n}\right\} $and $\left\{ Y_{1},...,Y_{n}\right\} $,

\[
U=\frac{1}{{ \choose {n}{r}}{ \choose {m}{s}}}\sum_{\left|\alpha\right|=s,\alpha\subset\left[m\right]}\sum_{\left|\beta\right|=r,\beta\subset\left[n\right]}h\left(X_{\beta},Y_{\alpha}\right)
\]

where $h:\ X^{r}\times Y^{s}\to\mathbb{R}$. $h$ is symmetric in
$X$ and $Y$ variables individually.\\
\end{definition}

\begin{example*}[Mann-Whitney Statistic]

Is $X$ stochastically dominated by Y? Or are the at the same location?

\begin{align*}
\theta\left(P\right) & =P\left(X\leq Y\right)\\
h\left(X,Y\right) & =\mathbf{1}\left\{ X\leq Y\right\} \\
U_{n,m} & =\frac{1}{nm}\sum_{i=1}^{n}\sum_{j=1}^{m}\mathbf{1}\left\{ X_{i}\leq Y_{j}\right\} 
\end{align*}
\\
\end{example*}

\subsection{Variance of U-Statistics}

This is a precursor to asymptotic normality because "1st order terms" dominate everything else.

\begin{definition}

Assume that $E\left[\left|h\right|^{2}\right]<\infty$ for any $c<r$.
Define 

\[h_{c}\left(X_{1},...,X_{c}\right):=E\left[h\left(\underset{\text{fixed}}{\underbrace{X_{1},...,X_{c}}},\underset{\text{i.i.d P}}{\underbrace{X_{c+1},...,X_{r}}}\right)\right].\]

\begin{remark}
\begin{enumerate}
\item $h_{0}=E\left[h\left(X_{1},...,X_{r}\right)\right]=\theta\left(P\right)$
\item $E\left[h_{c}\left(X_{1},...,X_{c}\right)\right]=E\left[h\left(X_{1},...,X_{r}\right)\right]=\theta\left(P\right)$
\end{enumerate}

\end{remark}

\end{definition}

\begin{definition}
\begin{align*}
\hat{h}_{c}: & =h_{c}-E\left[h_{c}\right]=h_{c}-\theta\left(P\right)\\
E\left[\hat{h}_{c}\right] & =0
\end{align*}

Then define

\[
\zeta_{c}:=\text{Var}\left(h_{c}\left(X_{1},...,X_{c}\right)\right)=E\left[\hat{h}_{c}^{2}\right]
\]
\end{definition}

\begin{paragraph}{Goal:}

Write $Var\left[U_{n}\right]$ in terms of $\zeta_{c}'s$ for $c=1,2,...,r$.
Note the following:

If $\beta=\left\{ i_{1},...,i_{r}\right\} $, $\alpha=\left\{ i_{1}^{\prime},...,i_{r}^{\prime}\right\} $,
$S=\alpha\cap\beta$, $\left|S\right|=c$, then

\[
E\left[\hat{h}\left(X_{\beta}\right)\hat{h}\left(X_{\alpha}\right)\right]=\zeta_{c}.
\]

\end{paragraph}

\end{document}


\end{document}