\documentclass[11pt]{article}

% This package contains many of the macros you may need.
%
% A useful environment defined in it is the example environment, which you can
% invoke via \begin{example} and \end{example}.
\usepackage{scribe}

\begin{document}

\begin{center}
  \fcolorbox{black}{white}{
    \begin{minipage}{.9\columnwidth}
      \textbf{Stats 300b: Theory of Statistics}
      \hfill
      \textbf{Winter 2017} \\
      \vspace{-.2cm}
      \begin{center}
        {\LARGE Lecture 18 -- March 9} \\
      \end{center}
      \emph{Lecturer: John Duchi}
      \hfill
      \emph{Scribe: Michael Celentano} 
    \end{minipage}
  }
\end{center}

\begin{center}
  \dbend ~~  \emph{\textbf{Warning:} these notes may contain factual errors}
\end{center}

\paragraph{Reading:} Van der Vaart, \emph{Asymptotic Statistics}, Chapter 6.

\paragraph{Outline:} 

\begin{enumerate}
\item Absolute continuity of measures.
\item Contiguity and Asymptotics; LeCam's Lemmas.
\item Exact Calculations with Testing.
\end{enumerate}

\section{Absolute Continuity}
We begin with the notion that allows us to change measures in a non-asymptotic setting.
\begin{definition}
Probability measure $Q$ is \emph{absolutely continuous} with respect to probability measure $P$, written $Q \ll P$, if $P(A) = 0$ implies $Q(A) = 0$ for any set $A$.
\end{definition}
If $Q \ll P$, then the Radon-Nikodym theorem says that there exists a nonnegative measurable function $g$, denoted $\frac{dQ}{dP}$, such that $\E_Q[f] = \E_P[fg] = \int fg dP = \int f \frac{dQ}{dP}dP$ for all $f$ integrable with respect to $Q$. The following corollary is another formulation of some of these ideas.

\begin{corollary}\label{non-asymptotic-expectation}
Let $M$ be the joint measure (law) of the pair $ (X,V) \defeq \left(X, \frac{dQ}{dP}\right) $ under distribution $P$. (So $M$ is defined on $\mathcal X \times \R_+$). Then $V \geq 0$, $\E_M[V] = 1$, and $Q(B) = \E_P\left[\indic{B}(X)\frac{dQ}{dP}\right] = \E_P[\indic{B}(X)V] = \int_{B \times \R_+} VdM(x,v)$.  
\end{corollary}


\paragraph{Idea:} If we know $\frac{dQ}{dP}$ and we can compute things like $Pf$, then we can compute $Qf$ too. Can we do this in an asymptotic sense?
We would like to find an asymptotic version of Corollary \ref{non-asymptotic-expectation}. 

\section{Contiguity and Asymptotics; LeCam's Lemmas.}

The notion that will allow to perform the asymptotic versions of the previous calculations is contiguity.

\begin{definition} A sequence $\{Q_n\}$ of distributions is \emph{contiguous} with respect to $\{P_n\}$, written $Q_n \triangleleft P_n$, if $P_n(A_n) \rightarrow 0$ implies $Q_n(A_n) \rightarrow 0$ for any sequence of sets $A_n$. Sequences $\{Q_n\}$ and $\{P_n\}$ are \emph{mutually contiguous}, written $Q_n \triangleleft \!\triangleright P_n$, if $Q_n \triangleleft P_n$ and $P_n \triangleleft Q_n$. 
\end{definition}

Below we will characterize contiguity with conditions on the limits of density representations of $P_n$ and $Q_n$. Because $P_n$ and $Q_n$ need not be absolutely continuous with respect to each other, nor are we a priori provided some mutually dominating measure, the following observations are useful.

\begin{observation}
Even if $Q_n \not \ll P_n$, we can still consider $Q_n = Q_n^\| + Q_n^\perp$ where $Q_n^\| \ll P$ and $Q_n^\perp \perp P$. If we define $\frac{dQ_n}{dP_n} \defeq \frac{dQ_n^\|}{dP_n}$, then $\frac{dQ_n}{dP_n} \geq 0$ and $\E_{P_n}\left[\frac{dQ_n}{dP_n}\right] = Q_n^\|(\Omega) = 1 - Q_n^\perp(\Omega) \leq 1$. Thus, under $P_n$, the sequence $\frac{dQ_n}{dP_n}$ is tight. 
\end{observation}

\begin{observation}
We can always assume without loss of generality that $P_n$ and $Q_n$ all have densities $p_n$ and $q_n$ with respect to some finite base measure $\mu$. One suitable measure is $\mu = \sum_{n=1}^\infty 2^{-n} (P_n + Q_n)$, which has total mass 2. 
We can also without loss of generality take $\frac{dQ_n}{dP_n} = \frac{q_n}{p_n}$, so that $\int \frac{q_n}{p_n}dP_n = \int q_n \indic{p_n > 0} d\mu = Q_n^\|(\Omega)$. 
\end{observation}

We are now ready to state alternative characterizations of contiguity. 

\begin{lemma}[LeCam's First Lemma, or ``Limits determine contiguity''] \label{LeCam-1}

The following are equivalent:
\begin{enumerate}
\item $Q_n \triangleleft P_n$.
\item If $\frac{dQ_n}{dP_n} \overset{d}{\underset{Q_n}\longrightarrow} U$ along a subsequence, then $\P(U > 0) = 1$.
\item If $\frac{dQ_n}{dP_n} \overset{d}{\underset{P_n}\longrightarrow} V$ along a subsequence, then $\E[V] = 1$. 
\item If $T_n \stackrel{P_n}\longrightarrow 0$, then $T_n \stackrel{Q_n}\longrightarrow 0$.
\end{enumerate}
\end{lemma}

\begin{proofof}{$(4) \Rightarrow (1)$}
Take $A_n$ such that $P_n(A_n) \rightarrow 0$. Define $T_n = \indic{A_n}$. Then certainly $T_n \stackrel{P_n}{\rightarrow} 0$, so $T_n \stackrel{Q_n}{\rightarrow}0$. That is, $Q_n(A_n) \rightarrow 0$.  
\end{proofof}
The remaining parts of the proof can be found in Van der Vaart Chapter 6.

\vspace{.2cm}

\noindent The following two examples will be useful in later developments.

\begin{example}[Asymptotic log normality]\label{asymptotic-log-normality}
Suppose we have $\log \frac{dP_n}{dQ_n} \overset{d}{\underset{Q_n}\longrightarrow} \normal(\mu,\sigma^2)$. Then, by the continuous mapping theorem, we have $\frac{dP_n}{dQ_n} \overset{d}{\underset{Q_n}\longrightarrow} \exp(\normal(\mu,\sigma^2))$ is greater than 0 with probability 1. Applying the second characterization of LeCam's First Lemma implies $Q_n \triangleleft P_n$. On the other hand, based on our knowledge of the moment generating function for normal random variables, $\E[\exp(\normal(\mu,\sigma^2)] = \exp\left(\mu + \frac12\sigma^2\right)$. 
Applying the third characterization of LeCam's First Lemma gives that $P_n \triangleleft Q_n$ if and only if $\exp\left(\mu + \frac12\sigma^2\right) = 1$. That is, $\mu = - \frac12 \sigma^2$. 
\end{example} 

\begin{example}[Smooth likelihoods]
Suppose $\{P_\theta\}_{\theta \in \Theta}$ has densities $p_\theta$, and $p_\theta$ is smooth enough in $\theta$ that $\log p_\theta$ has a Taylor expansion around $\theta_0 \in \interior \Theta$.
That is,
$$
\log p_{\theta_0 + h} = \ell_{\theta_0} + h^T \nabla \ell_{\theta_0} + \half h^T \nabla^2 \ell_{\theta_0} h + O(\norm{h}^3)
$$
where $\ell_\theta = \log p_\theta$. 

A Taylor expansion gives
\begin{align*}
\log\frac{p_{\theta_0+h/\sqrt n} (x_1,\ldots,x_n)}{p_{\theta_0}(x_1,\ldots,x_n)} &= \sum_{i=1}^n \log \frac{p_{\theta_0+h/\sqrt n}(x_i)}{p_{\theta_0}(x_i)} = \sum_{i=1}^n \left(\ell_{\theta_0 + h/\sqrt n}(x_i) - \ell_{\theta_0}(x_i)\right) \\
&= \frac1{\sqrt n} \sum_{i=1}^n h^T\nabla\ell_{\theta_0}(x_i) + \frac1{2n}h^T\sum_{i=1}^n \nabla^2 \ell_{\theta_0}(x_i)h + o_{P_{\theta_0}}(1)\\
&= \underbrace{\frac1{\sqrt n} \sum_{i=1}^n h^T\nabla\ell_{\theta_0}(x_i)}_{\stackrel{d}\rightarrow\normal(0,h^TI_\theta h)} - \half h^TI_{\theta_0}h + o_{P_{\theta_0}}(1)\\
&\stackrel{p}{\rightarrow} \normal\left(-\half h^T I_{\theta_0} h, h^TI_{\theta_0} h\right). 
\end{align*}
Surprisingly, this is exactly the condition for mutual contiguity from Example \ref{asymptotic-log-normality}! Thus $P_{\theta_0}^n \mathop{\triangleleft\!\triangleright} P_{\theta_0 + h/\sqrt n}^n$.  Local alternatives and the null are mutually contiguous!
\end{example}

In analogy with Corollary \ref{non-asymptotic-expectation} and in light of the previous example, we now wonder whether mutual contiguity tells us something about limit distributions under alternatives. The next theorem is the beginning of an answer.

\begin{theorem}[LeCam]\label{asymptotic-measure-change}
Let $P_n$, $Q_n$ be distributions on $X_n \in \R^d$. If $Q_n \triangleleft P_n$ and $\left(X_n, \frac{dQ_n}{dP_n}\right) \overset{d}{\underset{P_n}\longrightarrow} (X,V)$, then $L(B) \defeq \E[\indic{B}(X)V]$ is a probability measure (ie. $\E[V] = 1$ and $V \geq 0$) and $X_n \overset{d}{\underset{Q_n}\longrightarrow} W$ where $W \sim L$. 
\end{theorem}

\begin{proof}
(See Van der Vaart Chapter 6).
\end{proof}

The following example is an important application of Theorem \ref{asymptotic-measure-change}.

\begin{example}[LeCam's Third Lemma]
If $\left(X_n,\log\frac{dQ_n}{dP_n}\right) \overset{d}{\underset{P_n}\longrightarrow} \normal\left( \begin{pmatrix} \mu \\ - \half \sigma^2 \end{pmatrix}, \begin{pmatrix} \Sigma & \tau \\ \tau^T & \sigma^2 \end{pmatrix} \right)$, then $X_n \overset{d}{\underset{Q_n}\longrightarrow} \normal(\mu + \tau, \Sigma)$. The proof, which we omit, invovles a calculation with characteristic functions.
\end{example}

\section{Exact Calculations with Testing}
We briefly introduce our next question, which will be answered in future lectures. 

Consider the simple hypothesis test $P_0$ vs. $P_1$. The total variation distance comes up naturally in this context when considering optimal error rates. Indeed,
\begin{align*}
\inf_{\Phi:\mathcal X \rightarrow \{0,1\}} \big(P_0(\Phi \neq 0) + P_1(\Phi \neq 1)\big) &= \inf_A \{ P_0(A^c) + P_1(A)\} = \inf_A \{1 - (P_0(A) - P_1(A))\}\\
&= 1 - \tvnorm{P_0 - P_1}.
\end{align*}
Thus, given a sequence of nulls and alternatives $\{P_{0,n}\}$ and $\{P_{1,n}\}$, if $\tvnorm{P_{0,n} - P_{1,n}}$ converges to something in $(0,1)$, then it is impossible to test exactly whether the null or the alternative is true. The result of Problem Set 8, Question 2 implies that this also holds if $\dhel(P_{0,n},P_{1,n})$ converges to something in $(0,1)$.

We now ask: 
How/when do we have this type of limiting behavior? 

\end{document}



