\documentclass[11pt]{article}

% This package contains many of the macros you may need.
%
% A useful environment defined in it is the example environment, which you can
% invoke via \begin{example} and \end{example}.
\usepackage{scribe}
\usepackage{commath}

\newcommand{\contiguous}{\triangleleft}
\newcommand{\mutuallycontiguous}{\triangleleft \triangleright ~}
\newcommand{\distto}[1][]{\overset{d}{\underset{#1}{\to}}}
\newcommand{\tv}[2]{\left\| #1 - #2 \right\|_{\text{TV}}}
\newcommand{\hellinger}{d_{\text{hel}}}
\newcommand{\alternative}{h/\sqrt{n}}

\begin{document}

\begin{center}
  \fcolorbox{black}{white}{
    \begin{minipage}{.9\columnwidth}
      \textbf{Stats 300b: Theory of Statistics}
      \hfill
      \textbf{Winter 2017} \\
      \vspace{-.2cm}
      % Instead of Lecture 1, January 10, put the actual lecture number and
      % date of the lecture here
      \begin{center}
        {\LARGE Lecture 19 -- March 14} \\
      \end{center}
      \emph{Lecturer: John Duchi}
      \hfill
      \emph{Scribe: Steven Yadlowsky}  % You should put your name here if you scribe
    \end{minipage}
  }
\end{center}

\begin{center}
  \dbend ~~  \emph{\textbf{Warning:} these notes may contain factual errors}
\end{center}

\paragraph{Reading:}  Notes on course website (Contiguity and asymptotics)% Put the reading covered in the lecture here

%%%%%%%%%%%%%%%%% SCRIBE NOTES HERE %%%%%%%%%%%%%%%%%%%%
\section{Outline}
\begin{itemize}
  \item Recap
  \item Quadratic mean differentiability
    \begin{itemize}
      \item Testing
      \item Definitions and examples
    \end{itemize}
  \item Local asymptotic normality
  \item Limiting Gaussian shifts
\end{itemize}

\section{Recap}
Measures $Q_n$ are contiguous with respect to $P_n$ (written $Q_n \contiguous
P_n$) if $Q_n(A_n) \to 0$ whenever $P_n(A_n) \to 0$.

\begin{lemma}{Le Cam's 3rd Lemma}
  If
  \begin{equation*}
    \left( X_n, \log \frac{Q_n}{P_n} \right) \distto[P_n]
    \normal \left(
      \left[\begin{array}{c} \mu \\ -\frac{1}{2}\sigma^2 \end{array}\right],
      \left[\begin{array}{cc} \Sigma & \tau \\ \tau^T & \sigma^2 \end{array}\right]
    \right),
  \end{equation*}
  then $P_n \mutuallycontiguous Q_n$ and $X_n \distto[Q_n] \normal \left( \mu +
    \tau, \Sigma \right)$.
\end{lemma}

\paragraph{Idea} Asymptotically, we can change measure between $P_n$ and $Q_n$,
because $Q_n \mutuallycontiguous P_n$ and \\
$\log \frac{\dif{Q_n}}{\dif{P_n}} \to \normal\left( -\frac{1}{2}\sigma^2, \sigma^2 \right).$

\paragraph{Goal} Understand limiting behavior of experiments / random variables
by this change of measure. Our motiviation will be via testing: When is testing
point nulls versus point alternatives ``appropriated'' (which we will
momentarily make precise) hard?

\paragraph{Recall}
\begin{gather*}
  \tv{P}{Q} = \sup_A |P(A) - Q(A)|, \\
  \hellinger^2(P,Q) = \frac{1}{2} \int \left(\sqrt{p} - \sqrt{q}\right)^2
  \dif{\mu}, \\
  \hellinger^2(P,Q) \le \tv{P}{Q} \le \hellinger{P, Q} \sqrt{2 - \hellinger^2(P, Q)}.
\end{gather*}

Now, consider the simple hypothesis testing problem of $P_0$ versus $P_1$, and
the associated best error,
\begin{equation*}
\inf_{\psi} P_0(\psi \not= 0) + P_1(\psi \not= 1) = 1 - \tv{P_0}{P_1} \ge 1 -
\sqrt{2} \hellinger(P_0, P_1).
\end{equation*}
Given sequences of tests $P_{0,n}$ versus $P_{1,n}$, we are interested in
considering when the asymptotic error does not vanish,
\begin{equation*}
\liminf_{n \to \infty} \inf_{\psi_n} P_{0,n}(\psi_n \not= 0) + P_{1,n}(\psi_n \not= 1) > 0,
\end{equation*}
which occurs whenever $1 - \sqrt{2} \hellinger(P_{1,n}, P_{0,n}) > 0$, or, put
another way, when $\hellinger(P_{1,n}, P_{0,n}) < \frac{1}{\sqrt{2}}.$ Note that
this bound may very well be loose-- we can probably get tighter constant bounds
on the Hellinger distance, but that is not our intention here. Instead, we are
focusing on highlighting how Hellinger distance ``plays nicely'' with iid
sampling (ie., product distributions). Specifically, consider the following:
\begin{align*}
  \hellinger^2(P^n, Q^n) &= \frac{1}{2} \int \left( \sqrt{\dif{P^n}} -
                           \sqrt{\dif{P^n}} \right)^2 \\
  &= 1 - \int \sqrt{p(x_1)\dots p(x_n)}\sqrt{q(x_1)\dots q(x_n)} \dif\mu \\
  &= 1 - \left( \int \sqrt{p(x)}\sqrt{q(x)} \dif\mu \right)^n = 1 - \left(1 -
    \hellinger^2(P, Q)\right)^n.
\end{align*}
In particular, given $\hellinger(P, Q)$ we \emph{know} $\hellinger(P^n, Q^)$ of
the product distributions. So, if we consider local alternatives $P_0^n$ versus
$P_{h/\sqrt{n}}^n$, then
\begin{equation*}
  \lim_{n \to \infty} \hellinger(P_0^n, P_{h/\sqrt{n}}^n) < 1 \text{ if}
\end{equation*}
\begin{equation*}
  \hellinger^2(P_0, P_{h/\sqrt{n}}) = O(\frac{1}{n}) \text{ as } n \to \infty,
\end{equation*}
because $(1 - O(\frac{1}{n}))^n$ ``$\to$'' $\exp(-\text{something})$. With this
in mind, our game plan is to understand when $\hellinger^2(P_0, P_\alternative)
= \frac{f(h)}{n} + o(\frac{1}{n}).$

\section{Quadratic mean differentiability}
Suppose $\{ P_\theta \}_{\theta \in \Theta}$ is a smooth family of distributions
(ie., $\nabla p_\theta$ exists and is smooth). Then, using that
\[
  \sqrt{a+\delta} = \sqrt{a} + \frac{1}{2\sqrt{a}} \delta + O(\delta^2),
\]
as $\delta \to 0$, we have that
\begin{align*}
  \sqrt{p_{\theta+h}} &= \sqrt{p_{\theta} + \nabla p_\theta^T h + O(\|h\|^2)} \\
  &= \sqrt{p_{\theta}} + \frac{1}{2\sqrt{p_\theta}} \nabla p_\theta^T h +
    O(\|h\|^2) \\
  &= \sqrt{p_{\theta}} + \frac{1}{2} \frac{\nabla p_\theta^T
    h}{p_\theta}\sqrt{p_\theta} + O(\|h\|^2) \\
  &= \sqrt{p_{\theta}} + \frac{1}{2} \dot\ell_\theta^T
    h\sqrt{p_\theta} + O(\|h\|^2),
\end{align*}
where $\dot\ell_\theta = \nabla \log p_\theta$ is the score function.

With this in mind, we define the following nice family of distributions for
which the above expansion holds, almost definitionally. This will basically
capture families of distributions that have nice behavior with the Hellinger
distance.

\begin{definition}
  A family $\{ P_\theta \}_{\theta \in \Theta}$ is quadratic mean differentiable
  (QMD) at $\theta \in \operatorname{int}\Theta$ if there is a score function
  $\dot\ell_\theta : \mathcal{X} \to \R^d$, so that
  \[
    \int \left( \sqrt{p_{\theta+h}} - \sqrt{p_{\theta}} -
      \frac{1}{2}\dot\ell_\theta ^T h \sqrt{p_\theta} \right)^2 \dif{\mu} = o\left(\|h\|^2\right),
  \]
  as $h \to 0$.
\end{definition}

\begin{proposition}{(proved in the notes)} $P_\theta \dot\ell_\theta = 0$ and
  $P_\theta \dot\ell_\theta \dot\ell_\theta^T$ is well-defined in a family which
  is QMD.
\end{proposition}

\begin{example}{Exponential families.} Let $p_\theta(x) = \exp(\theta^T T(x) -
  A(\theta))$, $A(\theta) = \log \int \exp(T(x)^T\theta).$ Then, $\{P_\theta\}$
  is QMD with score $\dot\ell_\theta(x) = \nabla \log p_\theta(x) = T(x) -
  \nabla A(\theta) = T(x) - \E(T(x)).$

  \begin{proof}(Sketch).

    Without loss of generality, we can take $T(x) = x$ (we could do a change of
    measure with $\mu$ to make this precise).

    \begin{align*}
      &\sqrt{p_{\theta+h}} - \sqrt{p_{\theta}} - \frac{1}{2} h^T\left(x - \nabla
        X(\theta)\right) \sqrt{p_\theta} \\
      =& \exp\left( \frac{1}{2}(x^T \theta - A(\theta))\right) \left(\exp\left(
         \frac{1}{2} h^T x - \frac{1}{2}\left(A(\theta+h) - A(\theta) \right)\right) -
         1 - \frac{1}{2} h^T (x - \nabla A(\theta)) \right) \\
      =& \sqrt{p_\theta} \left( \frac{1}{2} h^T x - \frac{1}{2}h^T(x - \nabla
         A(\theta)) - \frac{1}{2}\left(A(\theta+h) - A(\theta) \right) + O\left(
         (h^T x - A(\theta + h) - A(\theta))^2 \right) \right)\\
      =& \sqrt{p_\theta} \left( \frac{1}{2}(A(\theta) + A(\theta+h)) + \frac{1}{2}h^T \nabla
         A(\theta) + O\left((h^T x)^2 + \|h\|^2 \right) \right) \\
      =& \sqrt{p_\theta} \left( -\frac{1}{2}h^T \nabla A(\theta) +
         \frac{1}{2}h^T \nabla A(\theta) + O\left((h^T x)^2 + \|h\|^2 \right)
         \right) \\
      =& \sqrt{p_\theta} \left(O\left((h^T x)^2 + \|h\|^2 \right) \right)
    \end{align*}
    Using that Lebesgue's dominated convergence theorem holds in an exponential
    family,
    \begin{align*}
      \frac{1}{\|h\|^2} \int \left( \sqrt{p_{\theta+h}} - \sqrt{p_\theta} -
      \frac{1}{2} h^T \dot\ell_\theta \sqrt{p_\theta}\right) &=
       \int p_\theta O\left(\frac{\|h\|^4 + (h^T x)^4}{\|h\|^2}\right) \dif{\mu} \\
      &\to 0 \text{ as }h\to 0.
    \end{align*}
  \end{proof}
\end{example}

\begin{remark}
  Fisher information $I_\theta = P_\theta \dot\ell_\theta \dot\ell_\theta^T$ is
  exactly as before, $I_\theta = \operatorname{Cov}(T(X) - \nabla A(\theta)).$

  The following heuristic (which is actually true, although unstated as such) is
  that if $\{ P_\theta \}$ is QMD, then
  \begin{align*}
    \hellinger^2(P_{\theta + h}, P_{\theta})
    &\underbrace{=}_{\text{can be made rigorous}} \frac{1}{2}\int
    \left(\frac{1}{2} h^T \dot\ell_theta \sqrt{p_\theta} \right)^2 \dif\mu +
      o(\|h\|^2) \\
    &= \frac{1}{8} \int h^T \dot\ell_\theta \dot\ell_\theta^T h p_\theta
      \dif{\mu} + o(\|h\|^2) \\
    &= \frac{1}{8} h^T I_\theta h + o(\|h\|^2).
  \end{align*}
\end{remark}

For QMD families, we have
\[\hellinger^2(P_{\theta+\alternative}, P_\theta) = \frac{1}{8}h^T I_\theta h +
  o(\frac{1}{n},\text{ and} \]
\[
  \lim_{n\to \infty} \hellinger^2(P_{\theta + \alternative}^n, P_\theta^n) = 1 -
  \exp(-\frac{1}{8} h^T I_\theta h) \in (0, 1),
\]
so that
\[
\liminf_{n \to \infty} \inf_{\psi_n} P_{\theta}(\psi_n \not= 0) + P_{\theta+\alternative}(\psi_n \not= 1) > 0.
\]

\section{Local asymptotic normality}

\begin{definition} A family $\{ P_\theta \}_{\theta \in \Theta}$ is locally
  asymptotically normal (LAN) at $\theta \in \operatorname{int} \Theta$ if there
  exists a sequence $D_n \in \R^d$ and precision matrix $K \succeq 0$, such that
  for all $h \in \R^d$,
  \[
    \log \frac{\dif{P_{\theta + \alternative, n}}}{\dif{P_{\theta, n}}} = h^T
    \Delta_n - \frac{1}{2} h^T K h + o_{P_{\theta, n}}(\|h\|),
  \]
  where $\Delta_n \distto[P_{\theta,n}] \normal(0, K)$, and $o_P(\|h\|)$ means
  converging in probability to 0 uniformly, if $\|h\|$ is bounded.
\end{definition}

\begin{remark}
  \begin{enumerate}
  \item Basically, $p_\theta$ has a shifted quadratic expansion.
  \item $h^T \Delta_n - \frac{1}{2} h^T K h \distto[P_{\theta, n}]
    \normal(-\frac{1}{2} h^T K h, h^T K t)$, so this will imply contiguity.
  \end{enumerate}
\end{remark}

\begin{example}{Gaussian shifts.} Let $P_{h,n}$ be  the distribution of $Y_i = h
  + \xi_i,$ where $\xi_i \sim \normal(0, \Sigma)$, and $i = 1,\dots n$.
  Then, calculations (omitted here) show that
  \[
    \log \frac{\dif{P_{\alternative, n}}}{\dif{P_{0, n}}} = \sqrt{n}h^T
    \Sigma^{-1} \bar Y_n - \frac{1}{2} h^T \Sigma^{-1} h,
  \]
  \noindent so
  \[
    \Delta_n = \sqrt{n} \Sigma^{-1} \bar Y_n \distto[P_{0, n}] \normal(0, \Sigma^{-1}),
  \]
  and $K = \Sigma^{-1}$ is the precision (or information) matrix.
\end{example}

\begin{example}{QMD families.} (see VdV ch 7 for details).

  If $\{P_\theta\}$ is QMD, then 
  \[
    \log \frac{\dif{P_{\alternative, n}}}{\dif{P_{0, n}}} = \left(
      \frac{1}{\sqrt{n}} \sum_{i=1}6n \dot \ell (x_i)\right)^T h - \frac{1}{2}
    h^T I_\theta h + o_P(1).
  \]
  So, QMD implies LAN with precision $I_\theta$.

  \begin{proof}(Sketch).

    Let $P_n = P_{\theta + \alternative}$, and $P = P_\theta$.
    \begin{align*}
      \log\prod_{i=1}^n \frac{p_n}{p}(x_i) &= 2 \sum_{i=1}^n \log
                                             \sqrt{\frac{p_n}{p}(x_i)}\\
      \left(\ast\right)&= 2 \sum_{i=1}^n \log\bigg( 1+ \frac{1}{2}\underbrace{\left(2
        \sqrt{\frac{p_n}{p}}(x_i) - 2\right)}_{W_{n,i}} \bigg) \\
      &= 2 \sum_{i=1}^n \frac{1}{2} W_{n,i} - \frac{1}{8} W_{n,i}^2 + W_{n,i}^2r(W_{n,i}),
    \end{align*}
    where $r(x) = O(|x|)$.
    \begin{align*}
      \left(\ast\right)&= \underbrace{\sum_{i=1}^n W_{n,i}}_{(1)} -
                         \underbrace{\frac{1}{4} \sum_{i=1}^n W_{n,i}^2}_{(2)} + o_P(1).
    \end{align*}
    From here, we can use QMD to control $(1)$ and $(2)$, and if $g(x)= h^T
    \dot\ell_\theta(x)$, then
    \begin{align*}
      \var\left(\sum_{i=1}^n W_{n,i} - \frac{1}{\sqrt{n}} \sum_{i=1}^n g(X_i)
      \right) &\le n \E\left[(W_{n,1} - \frac{1}{\sqrt{n}} g(X_1))^2 \right] \\
      &= n o(\frac{1}{n}) = o(1),
    \end{align*}
    where the step to $o(\frac{1}{n})$ follows from the fact that the family is QMD.

    Similar calculations show that in the end,
    \[
      (\ast) = \frac{1}{\sqrt{n}} \sum_{i=1}^n g(X_i) - \frac{1}{2} h^T I_\theta
      h + o_P(1).
    \]
  \end{proof}
\end{example}

\end{document}
