\documentclass[11pt]{article}

% This package contains many of the macros you may need.
%
% A useful environment defined in it is the example environment, which you can
% invoke via \begin{exmp} and \end{exmp}.
\usepackage{xcolor}
\usepackage{manfnt}
\usepackage{scribe}
\usepackage{amsmath}              
\usepackage{amsthm}   
\newtheorem{thm}{Theorem}
\newtheorem{prop}[thm]{Proposition}
\newtheorem{dfn}{Definition}[section]
\newtheorem{lem}[thm]{Lemma}
\newtheorem{exmp}{Example}

\usepackage{amsmath,amsfonts,amssymb,graphicx,mathtools,flexisym}

\newcommand\BB{\mathcal{B}}
\newcommand\EE{\mathbb{E}}
\newcommand\FF{\mathcal{F}}
\newcommand\GG{\mathcal{G}}
\newcommand\HH{\mathcal{H}}
\newcommand\II{\ell}
\newcommand\LL{\mathcal{L}}
\newcommand\MM{\mathcal{M}}
\newcommand\PP{\mathcal{P}}
\newcommand\RR{\mathcal{R}}
\newcommand\TT{\mathcal{T}}

\newcommand\covAS{\xrightarrow{as}}
\newcommand\covD{\leadsto}
\newcommand\covP{\xrightarrow{P}}
\newcommand\D{\mathbb{D}}
\newcommand\dlt{\delta}
\newcommand\Dlt{\Delta}
\newcommand\DT{\nabla}
\newcommand\eps{\varepsilon}
\newcommand{\iid}{\mathbin{\overset{iid}{\kern\z@\sim}}}
\newcommand\ind{\mathbbm{1}}
\newcommand\la{\leftarrow}
\newcommand\lmb{\lambda}
\newcommand\Lmb{\Lambda}
\def\NN{\mathcal{N}}
\newcommand\om{\omega}
\newcommand\Om{\Omega}
\newcommand\ra{\rightarrow}
\def\S{\mathcal{S}}
\newcommand\sd{\text{sd}}
\newcommand\sg{\sigma}
\newcommand\Sg{\Sigma}
\newcommand\sgn{\text{sgn}}
\def\t{\theta}
\newcommand\T{\Theta}
\newcommand\vt{\vartheta}
\newcommand{\vertii}[1]{{\left\vert\kern-0.25ex\left\vert #1 
    \right\vert\kern-0.25ex\right\vert}}
\newcommand{\vertiii}[1]{{\left\vert\kern-0.25ex\left\vert\kern-0.25ex\left\vert #1 
    \right\vert\kern-0.25ex\right\vert\kern-0.25ex\right\vert}}
\begin{document}


\begin{center}
  \fcolorbox{black}{white}{
    \begin{minipage}{.9\columnwidth}
      \textbf{Stats 300b: Theory of Statistics}
      \hfill
      \textbf{Winter 2017} \\
      \vspace{-.2cm}
      % Instead of Lecture 1, January 10, put the actual lecture number and
      % date of the lecture here
      \begin{center}
        {\LARGE Lecture 5 -- January 24} \\
      \end{center}
      \emph{Lecturer: John Duchi}
      \hfill
      \emph{Scribe: Amanda Zhang}  % You should put your name here if you scribe
    \end{minipage}
  }
\end{center}

\begin{center}
  \dbend ~~  \emph{\textbf{Warning:} these notes may contain factual errors}
\end{center}

\paragraph{Reading:} VDV 4 % Put the reading covered in the lecture here

%%%%%%%%%%%%%%%%% SCRIBE NOTES HERE %%%%%%%%%%%%%%%%%%%%
The method of moments determines estimators by comparing sample and theoretical moments. Let $X_1,\cdots, X_n$ be a sample from a distribution $P_\theta$ that depends on a parameter $\theta$, ranging over some set $\Theta$. Given $f: \mathcal{X}\ra \R^d$ with $P_{\theta_0}\vertii{f}_2^2<\infty$. By central limit theorem, 
\begin{equation}
\sqrt{n}(P_n f - P_{\theta_0} f)\covD N\left(0,\cov_{\theta_0}(f)\right).
\end{equation}
Let $e: \Theta \ra \R^d$ be the vector-valued expectation $e(\theta) = P_\theta f$. If $e$ is ``nice'' in that $e^{-1}(P_{\theta_0} f) ={\theta_0}$. Then by delta method,
$$\sqrt{n}\left( e^{-1} (P_n f) - e^{-1}(P_{\theta_0} f)\right) = \sqrt{n}\left( e^{-1} (P_n f) - {\theta_0} \right) \covD \left(e(P_{\theta_0} f)'\right)^{-1} N \left(0, \cov_{\theta_0} (f)\right).$$
\begin{thm}{ \textbf{inverse function theorem}} Let $F: \R^d \ra \R^d$ be continuously differentiable in a neighborhood of $\theta \in \R^d$, where $F'(\theta)$ is invertible, that is, $\det(F'(\theta))\neq 0$. Then in a neighborhood of $t= F(\theta)$, we have 
\begin{equation}
(F^{-1})'(t) = \frac{\partial}{\partial t} F'(t) = \left[F'\left(F^{-1}(t)\right)\right]^{-1}
\end{equation}
and $(F^{-1})'$ is continuous.
\end{thm}

\begin{thm}\label{thm4.2}
Suppose that $e(\theta)= P_\theta f$ is one-to-one on an open set $\Theta\subset \R^d$ and continuously differentiable at $\theta_0$ with nonsingular derivative $e_{\theta_0}'$. Moreover, assume that $P_{\theta_0} \vertii{f}_2^2<\infty$. Then moment estimators $\hat{\theta}_n$ exist with probability tending to one and satisfy 
\begin{equation}
\sqrt{n}(\hat{\theta}_n - \theta_0)\covD N\left( 0, (e(\theta_0)')^{-1} P_{\theta_0} ff^T\left((e(\theta_0)')^{-1} \right)^T\right)
\end{equation}
\end{thm}
\begin{proof}
Continuous differentiability at $\theta_0$ presumes differentiability in a neighborhood and the continuity of $\theta \mapsto e_\theta'$ and non singularity of $e_{\theta_0}'$ imply non-singularity in a neighborhood. Therefore, by the inverse function theorem, there exist open neighborhoods $U$ of $\theta_0$ and $V$ of $P_{\theta_0} f$ such that $e: U\mapsto V$ is a differentiable bijection with a differentiable inverse $e^{-1}: V\mapsto U$. Moment estimators $\hat{\theta}_n = e^{-1}(P_nf)$ exist as soon as $P_n f\in V$, which happens with probability tending to 1 by the law of large numbers. We know, by central limit theorem, that
\begin{equation}
\sqrt{n}(P_n f - P_{\theta_0} f)\covD N\left(0,\cov_{\theta_0}(f)\right).
\end{equation}
Apply Delta Method, we get:
\begin{equation}
\sqrt{n}\left( e^{-1} (P_n f) - e^{-1}(P_{\theta_0} f)\right) = \sqrt{n}(\hat{\theta}_n - \theta_0)\covD N\left( 0, (e(\theta_0)')^{-1} P_{\theta_0} ff^T\left((e(\theta_0)')^{-1} \right)^T\right).
\end{equation}
\end{proof}
\begin{exmp}
Let $X_i$ be i.i.d Bernoulli$\{\pm 1\}$ random variables. Then 
$$P_\theta (X = x) = \frac{e^{\theta x}}{1+e^{\theta x}} = \frac{1}{1+e^{-\theta x}}.$$
$$e(\theta) = \E_\theta[X] = \frac{1}{1+e^{-\theta}} - \frac{1}{1+e^\theta} = \frac{e^\theta -1}{e^\theta +1}.$$
Then $e^{-1}(t) = \log \frac{1+t}{1-t}$ and $e'(\theta) = \frac{e^\theta}{e^\theta +1} - \frac{e^{2\theta}}{(e^\theta +1)^2} = \frac{e^\theta}{(1+e^\theta)^2} = P_\theta(1-P_\theta).$ In particular, $(e'(\theta))^{-1} = \frac{1}{P_\theta(1-P_\theta)}$. The covariance $\cov_\theta(x)$ is $4P_\theta(1-P_\theta).$ Applying Theorem \ref{thm4.2}, we get
$$\sqrt{n}\left(e^{-1}(\overline{X_n} - \theta)\right) \covD N(0, \frac{4}{P_\theta(1-P_\theta)}).$$
\end{exmp}
\section{Exponential Family}
Given a measure $\mu$, we define an \emph{exponential family} of probability distributions as those distributions whose density (relative to $\mu$ have the following general form:
\begin{equation}\label{4.6}
p(x|\theta) = h(x)\exp[\theta^T T(x) - A(\theta)],
\end{equation}
for a parameter vector $\theta$, often referred to as the \emph{canonical parameter}, and for given functions $T$ and $h$. The statistic $T(X)$ is referred to as a \emph{sufficient statistic}; the function $A(\theta)$ is known as the \emph{cumulant function}. Integrating (\ref{4.6}) with respect to the measure $\mu$, we have
\begin{equation}\label{4.7}
A(\theta) = \log \int h(x) \exp[\theta^T T(x)]\mu(dx).
\end{equation}
The set of parameters $\theta$ for which the integral in (\ref{4.7}) is finite is referred to as the \emph{natural parameter space}: 
\begin{equation}
\mathcal{N} = \{\theta: \int h(x)\exp \{\theta^T T(x)\} \mu(dx)<\infty\}.
\end{equation}
We will restrict ourselves to exponential families for which the natural parameter space is a nonempty open set. Such families are referred to as \emph{regular}.
\\\\
\begin{prop}
$A(\theta)$ is convex and infintiely differentiable.
\end{prop}
As a consequence, we can calculate expectation and variance by differentiating $A$ with respect to $\theta$:
$$
\begin{aligned}
\frac{\partial A}{\partial \theta^T} &=\frac{\int T(x) \exp \{\theta^T T(x)\} h(x)\mu(dx)}{\int \exp \{\theta T(x)\} h(x) \mu(dx)}\\
& = \int T(x) \exp \{\theta^T T(x) - A(\theta)\} h(x)\mu(dx)\\
& =\E[T(x)].
\end{aligned}
$$
$$
\begin{aligned}
\frac{\partial^2 A}{\partial \theta \partial \theta^T} &= \int T(x) (T(x) - \frac{\partial}{\partial \theta^T} A(\theta))^T \exp\{\theta^T T(x) - A(\theta)\} h(x) \mu (dx)\\
&= \int T(x) (T(x) -\E[T(x)]^T \exp\{\theta^T T(x) - A(\theta)\}h(x)\mu(dx) \\
& = \E[T(X)T(X)^T] - \E[T(X)] \E[T(X)]^T \\
& = \var[T(X)].
\end{aligned}
$$
In general, higher-order moments of sufficient statistic can be obtained by taking higher-order derivatives of $A$.
\\\\
With the above techniques, it's not hard to obtain maximum likelihood estimates of the mean parameter in exponential family distributions. Consider an i.i.d. data set, $S= \{X_1,\cdots, X_n\}$. The log likelihood is:
\begin{equation}
\ell(\theta| S) = \log \left(\prod_{i=1}^n h(X_i) \right) +\theta^T \left(\sum_{i=1}^n T(X_i)\right) - nA(\theta).
\end{equation}
Taking the graduate with respect to $\theta$ yields:
\begin{equation}
\DT_\theta \ell = \sum_{i=1}^n T(X_i) - n\DT_\theta A(\theta),
\end{equation}
and setting it to zero gives:
\begin{equation}
\DT_\theta A(\hat{\theta}) = \frac{1}{n}\sum_{i=1}^n T(X_i).
\end{equation}
Finally, defining $\mu=\E[T(X)]$, we obtain
\begin{equation}
\hat{\mu}_{ML} = \frac{1}{n}\sum_{i=1}^n T(X_i)
\end{equation}
as the general formula for maximum likelihood estimation of the mean parameter in the exponential family.
\begin{thm}\label{thm4.3}
Let $\Theta \subset \R^d$ be open. Let the (exponential) family of densities $p_\theta$ be given by (\ref{4.6}) and be of full rank, meaning $\cov_\theta(T)>0$. Then the likelihood equation $\DT_\theta A(\hat{\theta}) = \frac{1}{n}\sum_{i=1}^n T(X_i)$ has a unique solution $\hat{\theta}_n$ with probability tending to $1$ and 
\begin{equation}\label{4.13}
\sqrt{n}(\hat{\theta}_n - \theta_0)\covD_{P_{\theta_0}} N\left(0, \DT^2 A(\theta_0)^{-1}\right)
\end{equation}
\end{thm}
\begin{proof}
By central limit theorem, we know 
$$\sqrt{n}(P_nT - P_{\theta_0}T) \covD N\left(0, \cov_{\theta_0} (T)\right).$$
Define $e(\theta) = P_\theta T$ as before. Then $e(\theta) = P_n T = \DT A(\theta)$ and $\left(e(\theta_0)'\right)^{-1} = \left( \DT^2 A(\theta_0)\right)^{-1} .$ Since $\cov_{\theta_0}(T) = \DT^2 A(\theta_0)^{-1}$, apply Theorem \ref{thm4.2} and (\ref{4.13}) follows.
\end{proof}
{\textbf{Remark:}} in exponential family, Fisher information $$I(\theta) = \E_\theta[\DT \ell_\theta \DT\ell_\theta^T] = \cov_\theta(T) = \DT^2 A(\theta).$$
\begin{exmp}[Linear Regression] Let $(x,y)\in \R^d \times R$ be i.i.d samples with density 
$$p_\theta(y|x) = \exp\left(-\frac{1}{2}(x^T\theta - y)^2\right),$$
where $Y|X=x$ follows $N(\theta^Tx,1)$. Then $L_n (\theta) = \sum_{i=1}^N \log P_\theta(y_i|x_i) =-\frac{1}{2} \vertii{x_\theta - y_\theta}_2^2$ and $\hat{\theta}_n = \argmax_{\theta}\vertii{X_\theta - Y}_2^2 = (X^TX)^{-1}X^TY$. Furthermore, $\ell_\theta(Y|X=x) = -\frac{1}{2}(x^T\theta -y)^2\Rightarrow \DT \ell_\theta(y|X=x) = x(x^T\theta -y) \Rightarrow \DT^2 \ell_\theta = xx^T.$ Thus, $I(\theta) = \E[XX^T]$. Apply Theorem \ref{thm4.3}, we obtain
$$\sqrt{n}(\hat{\theta}_n - \theta)\covD N(0,\E[XX^T]^{-1}).$$
\end{exmp}
\begin{dfn}{ \textbf{efficient}}
We say an estimator $\hat{\theta}_n$ is efficient for a parameter $\theta$ in model $\{P_\theta\}$ if 
$$\sqrt{n}(\hat{\theta}_n - \theta)\covD_{P_\theta} N(0, I_\theta^{-1}).$$

\end{dfn}
\begin{dfn}{ \textbf{asymptotic relative efficiency (ARE)}}
Let $\hat{\theta}_n$ and $T_n$ be estimators of parameter $\theta \in \R$. Assume that 
\begin{equation}
\sqrt{n}(\hat{\theta}_n - \theta) \covD N(0,\sigma^2(\theta)).
\end{equation}
Let $m(n)\ra \infty$ such that 
\begin{equation}
\sqrt{n}(T_{m(n)} - \theta)\covD N(0,\sigma^2(\theta)).
\end{equation}
The asymptotic relative efficiency of $\hat{\theta}_n$ with respect to $T_n$ is
\begin{equation}
\lim_{n\ra \infty} \inf \frac{m(n)}{n}.
\end{equation}
\end{dfn}
The intuition here is if $ARE = c\gg 1$, then $T_n$ requires sample size $C_n \gg n$ to get estimate of quality as $\hat{\theta}_n$. We can also see the interpretation through confidence interval: if ARE of $\hat{\theta}_n$ vs $T_n$ is $c$, then the asymptotic $1-\alpha$ confidence interval fro $\theta$ take $Z_{1-\alpha/2}$ such that 
$$Pr\left( |Z| \geq Z_{1-\alpha/2}\right) = \alpha,$$
where $\alpha \sim N(0,1).$ The confidence intervals of $\hat{\theta}_n$ and $T_n$ are:
$$C_{\hat{\theta}_n}: \left(\hat{\theta}_n - Z_{1-\alpha/2}\sqrt{\frac{\alpha^2}{n}}, \hat{\theta}_n + Z_{1-\alpha/2}\sqrt{\frac{\alpha^2}{n}}\right);$$
$$C_{T_n}: \left(T_n - Z_{1-\alpha/2}\sqrt{\frac{\frac{m(n)}{n}\sigma^2}{n}}, T_n + Z_{1-\alpha/2}\sqrt{\frac{\frac{m(n)}{n}\sigma^2}{n}}\right).$$ Then we have
$$\lim_{n\ra \infty} P_\theta(\theta \in C_{\hat{\theta}_n}) = \lim_{n\ra \infty} P_\theta(\theta \in C_{T_n}) = 1-\alpha.$$
Furthermore, 
$$\frac{\text{length}C(C_{T_n})}{\text{length}C(\hat{\theta}_n)} = \sqrt{\text{ARE}} \text{ of $\hat{\theta}_n$ with respect to $T_n$} =\sqrt{\frac{m(n)}{n}}.$$
\begin{prop}
Suppose $\hat{\theta}_n$ and $T_n$ are estimators of $\theta$ such that
$$\sqrt{n}(\hat{\theta}_n - \theta)\covD N(0, \sigma^2(\theta));$$
$$\sqrt{n}(T_n - \theta)\covD N(0, \tau^2(\theta)).$$
Then the ARE of $\hat{\theta}_n$ with respect to $T_n$ is $\frac{\tau^2(\theta)}{\sigma^2(\theta)}$. (In higher dimensions, it is roughly Tr($\tau^2(\theta)(\sigma^2(\theta)^{-1})$.
\end{prop}
\begin{proof}
Let $m(n) = \lceil \frac{\tau^2}{\sigma^2} \cdot n\rceil$. Then
$$
\begin{aligned}
& \sqrt{n}\left( T_{m(n)} - \theta\right)\\
= & \underbrace{\sqrt{\frac{n}{m(n)}}}_{\ra\frac{\sigma}{\tau}}\underbrace{ \sqrt{m(n)}\left(T_{m(n)} - \theta\right)}_{\covD N(0,\tau^2)}  \covD N(0,\sigma^2(\theta)).
\end{aligned}
$$
Thus, ARE is $\frac{m(n)}{n} = \frac{\tau^2}{\sigma^2}$.
\end{proof}
If $\tau^2 >\sigma^2$, we prefer $\hat{\theta}_n$ over $T_n$, because $T_n$ requires $\frac{\tau^2}{\sigma^2}$ times the sample size $\hat{\theta}_n$ does for the similar quality.


\end{document}