\documentclass[11pt]{article}
\usepackage{hyperref}
\newcommand{\gl}{\nabla l_{\theta_0}}
\newcommand{\gnl}{\nabla l_{\hat \theta_n}}
\newcommand{\gsl}{\nabla^2 \ell}
\newcommand{\cX}{\mathcal{X}}

% This package contains many of the macros you may  need.
%
% A useful environment defined in it is the example environment, which you can
% invoke via \begin{example} and \end{example}.
\usepackage{scribe}

\begin{document}

\begin{center}
  \fcolorbox{black}{white}{
    \begin{minipage}{.9\columnwidth}
      \textbf{Stats 300b: Theory of Statistics}
      \hfill
      \textbf{Winter 2017} \\
      \vspace{-.2cm}

      \begin{center}
        {\LARGE Lecture 4 -- January 19} \\
      \end{center}
      \emph{Lecturer: John Duchi}
      \hfill
      \emph{Scribe: Xiaotong Suo }  % You should put your name here if you scribe
    \end{minipage}
  }
\end{center}

\begin{center}
  \dbend ~~  \emph{\textbf{Warning:} these notes may contain factual errors}
\end{center}

\paragraph{Reading:}  VDV Chapter 3% Put the reading covered in the lecture here

%%%%%%%%%%%%%%%%% SCRIBE NOTES HERE %%%%%%%%%%%%%%%%%%%%
\paragraph{Outline of the lecture:}
\begin{enumerate}[I]
\item Asymptotic Normality \& Fisher information 
  \begin{enumerate}
  \item Basic Asymptotic Normality result
  \item Fisher information 
    \begin{enumerate}
    \item Definitions, Examples
    \item Information Inequality (Cramer Rao Bound)
    \end{enumerate}
  \end{enumerate}
\end{enumerate}

\section{The basic normality result}

As in the previous lecture, we assume as always that we have a model family
$\{P_\theta\}_{\theta \in \Theta}$, each distribution $P_\theta$ having
density $p_\theta$ with respect to some base measure $\mu$ on $\mc{X}$.  We
also use our usual notation that $\ell_\theta(x) \defeq \log p_\theta(x)$ is
the log-likelihood.  In order to get our asymptotic normality results, we
require a number of conditions on the smoothness of the log-likelihood so as
to perform appropriate Taylor expansions.  Recall briefly that if a function
$f : \R^d \to \R$ that is $k$-times continuously differentiable, then
\begin{align*}
  f(x + v)
  & =
  f(x) + \nabla f(x)^T v
  + \half v^T \nabla^2 f(x) v
  + \cdots + {\rm Rem}(x + v) [v^{\otimes k}],
\end{align*}
where $v^{\otimes k}$ indicates the $k$-th order tensor of $v$, i.e.\ the
tensor in $\R^{n^k}$ indexed by
$[v^{\otimes k}]_{i_1, \ldots, i_k} = v_{i_1} \cdots v_{i_k}$, and ${\rm Rem}$
is a remainder function such that
${\rm Rem}(x + v)$ acts linearly on the argument $v^{\otimes k}$ and
${\rm Rem}(x + v) \to 0$ as $v \to 0$. In some instances, we may say stronger
things, such as if the $(k-1)$th derivative is Lipschitz. To keep things
concrete, suppose $\nabla^2 f$ is Lipschitz, meaning that
$\opnorm{\nabla^2 f(x) - \nabla^2 f(y)} \le M \norm{x - y}$ for some
$M < \infty$.  In this case, we may take the remainder term to satisfy
$\opnorm{{\rm Rem}(x + v)} \le M \norm{v}$.

With these preliminaries out of the way, we begin with the major theorem we
would like to prove, which is that so long as the log likelihood
$\ell_\theta(x) \defeq \log p_\theta(x)$ is suitably smooth and that the
MLE $\what{\theta}_n$ is consistent, then
$\what{\theta}_n$ is asymptotically normal.
\begin{theorem}
  Let $X_i \simiid P_{\theta_0}$ where $\theta_0 \in \interior \Theta$.
  Assume that $\ell_\theta(x) = \log p_\theta(x)$ is smooth enough that
  $\E_{\theta_0}[\nabla \ell_{\theta_0}\nabla \ell_{\theta_0}^T]$ exists
  and that the Hessian $\nabla^2 \ell_\theta(x)$ is $M(x)$-Lipschitz in
  $\theta$, that is,
  \begin{equation*}
    \opnorm{\nabla^2 \ell_{\theta_1}(x) - \nabla^2 \ell_{\theta_2}(x)}
    \le M \norm{\theta_1 - \theta_2},
  \end{equation*}
  where $\E_{\theta_0}[M(X)^2]< \infty$. Assume
  additionally that the MLE
  $\what{\theta}_n$ is consistent, $\what{\theta}_n \cp \theta_0$.
  \begin{equation*}
    \sqrt n(\hat \theta_n - \theta_0) \cd \normal(0, I_{\theta_0}^{-1})
  \end{equation*}
  where $I_\theta = \E_\theta [\nabla \ell_{\theta}\nabla \ell_{\theta}^T]$ is
  the Fisher information.
\end{theorem}

\begin{proof}
  Let $\what{r}(x) \in \R^{d \times d}$ be the remainder matrix in
  Taylor expansion of the gradients of the individual log likelihood terms
  around $\theta_0$ guaranteed by Taylor's theorem (which certainly
  depends on $\what{\theta}_n - \theta_0$), that is,
  \begin{equation*}
    \nabla \ell_{\what{\theta}_n}(x)
    = \nabla \ell_{\theta_0}(x) + \nabla^2 \ell_{\theta_0}(x) (\what{\theta}_n
    - \theta_0)
    + \what{r}(x)(\what{\theta}_n - \theta_0),
  \end{equation*}
  where by Taylor's theorem
  $\opnorm{\what{r}(x)} \le M(x) \norms{\what{\theta}_n - \theta_0}$. Writing
  this out using the empirical distribution and that
  $\what{\theta}_n = \argmax_{\theta} P_n\ell_\theta(X)$, we have
  \begin{equation}
    \nabla P_n\ell_{\what{\theta}_n} = 0
    =P_n \nabla \ell_{\theta_0} 
    + P_n \nabla^2 \ell_{\theta_0} (\what{\theta}_n - \theta_0)
    + P_n \what{r}(X) (\what{\theta}_n - \theta_0).
    \label{eqn:star}
  \end{equation}
  But of course, expanding the term $P_n \what{r}(X) \in \R^{d \times d}$, we find that
  \begin{equation*}
    P_n \what{r}(X)
    = \frac{1}{n} \sum_{i = 1}^n \what{r}(X_i)
    ~~ \mbox{and} ~~
    \opnorm{P_n\what{r}}
    \le \underbrace{\frac{1}{n} \sum_{i = 1}^n M(X_i)}_{\cas
      \E_{\theta_0}[M(X)]}
    \underbrace{\norms{\what{\theta}_n - \theta_0}}_{\cp 0}
    = o_P(1).
  \end{equation*}
  In particular, revisiting expression~\eqref{eqn:star}, we have
  \begin{align*}
    0
    & = P_n \nabla \ell_{\theta_0} 
      + P_n \nabla^2 \ell_{\theta_0} (\what{\theta}_n - \theta_0)
      + o_P(1) (\what{\theta}_n - \theta_0). \\
    & = P_n \nabla \ell_{\theta_0} 
      + \left(P_{\theta_0} \nabla^2 \ell_{\theta_0}
      + (P_n - P_{\theta_0})\nabla^2 \ell_{\theta_0}
      + o_P(1)\right)(\what{\theta}_n - \theta_0).
  \end{align*}
  The strong law of large numbers guarantees that
  $(P_n - P_{\theta_0}) \nabla^2 \ell_{\theta_0} = o_P(1)$, and multiplying
  each side by $\sqrt{n}$ yields
  \begin{equation*}
    \sqrt{n} (P_{\theta_0} \nabla^2 \ell_{\theta_0} + o_P(1)) (\what{\theta}_n
    - \theta_0)
    = -\sqrt{n} P_n \nabla \ell_{\theta_0}.
  \end{equation*}

  Applying Slutsky's theorem gives the result:
  indeed, we have $T_n = \sqrt{n} P_n \nabla \ell_{\theta_0}$
  satisfies $T_n \cd \normal(0, I_{\theta_0})$ by
  the central limit theorem,
  and noting that $P_{\theta_0} \nabla^2 \ell_{\theta_0} + o_P(1)$ is
  eventually
  invertible gives
  \begin{align*}
    \sqrt{n} (\what{\theta}_n - \theta_0)
    \cd \normal(0, (P_{\theta_0}\gsl_{\theta_0})^{-1}I_{\theta_0} (P_{\theta_0}\gsl_{\theta_0})^{-1})
  \end{align*}
  as desired.
\end{proof}

\section{Fisher Information}
\begin{definition} For a model family $\{P_{\theta}\},\theta\in
  \Theta$ on $\mathcal{X}$. The fisher information is $I_\theta=
  I(\theta) =\E_\theta[\nabla l_\theta \nabla l_\theta^T] = \cov_\theta(\nabla l_\theta)$. When $\nabla$ and $\E$ are interchangable, then 
  \[I_\theta = -\E[\nabla^2 \log P_\theta(x)]\]
\end{definition}
\begin{example}
  Normal location family. $\{\normal(\theta,\sigma^2)\}_{\theta \in \reals}$, where $\theta$ is unknown, 
  \[
    \frac{\partial}{\partial\theta}\log P_\theta(x)  = \frac{\theta-x}{\sigma^2}. 
  \]
  Thus, 
  \begin{align*}
    \E[(\frac{\partial}{\partial\theta}\log P_\theta(x))^2]= \frac{\var(X)}{\sigma^4}
    =\frac{1}{\sigma^2}
  \end{align*}
  Heuristically speaking, if $\sigma^2\rightarrow 0$, then it's easy to estimate the mean. 
  If $\sigma^2\rightarrow \infty$, then it's hard to estimate $\theta$ because heavy tails. So fisher information roughly tells us how easy or hard to estimate a parameter. 
\end{example}

\begin{remark}
  What if we care about $\tau = h(\theta)$ instead of $\theta$? 
  Then inverse function theorem yields: 
  \begin{align*}
    \frac{\partial }{\partial \tau} h^{-1}(\tau) \bigg|_{\tau=h(\theta)} = \frac{1}{h' (h^{-1}(h(\theta)))} = \frac{1}{h'(\theta)}
  \end{align*}
  Therefore, we have 
  \begin{align*}
    I(\tau) = I(h(\theta)) = \frac{I(\theta)}{h'(\theta)^2}
  \end{align*}
  when $h'(\theta) \neq 0$. We can see this using the chain rule: 
  \begin{align*}
    \frac{\partial}{\partial \tau}\log P_{h^{-1}(\tau)} &= \frac{\partial }{\partial \tau}\log P_\theta  \\ &= \frac{\partial \log P_\theta}{\partial \theta}\frac{\partial \theta}{\partial \tau} \\&=\frac{\partial \log P_\theta}{\partial \theta}\frac{\partial h^{-1}(\tau)}{\partial \tau}
\end{align*}

\end{remark}

\begin{example}
Normal location $h(\theta) = \theta^2$. 
$h'(\theta) = 2\theta$, so 
\begin{align*}
I(\theta^2) = \frac{1}{4\theta^2}I(\theta) = \frac{1}{4\theta^2\sigma^2}
\end{align*}
In particular, as $\theta \rightarrow 0$, $I(\theta)\rightarrow \infty$. 
Suppose $\theta=0$, let $\hat \theta_n = (\frac{1}{n}\sum_{i =1}^n x_i)^2$, then 
\begin{align*}
n (\frac{1}{n}\sum_{i=1}^n x_i)^2 = (\frac{1}{\sqrt n}\sum_{i =1}^n x_i)^2 \cd z^2   
\end{align*}
where $z\sim N(0, \sigma^2)$. 
Therefore, we have an order of $n$ convergence. In this case, our estimator converges faster than $\sqrt n$. So heuristically speaking, if we have a higher fisher information, our estimator is somehow better. 
\end{example}
\paragraph{Additivity Property of Fisher information}
If $x_1\sim P_\theta$, $x_2 \sim Q_\theta$, $x_1, x_2$ independent, then $I_{x_1, x_2}(\theta) = I_{x_1}(\theta)+I_{x_2}(\theta)$. 

\begin{proof}
Since $x_1$ and $x_2$ are independent, 
\begin{align*}
\cov(\nabla \log P_\theta(x_1) + \nabla \log q_\theta(x_2)) = \cov(\nabla \log P_\theta(x_1)) + \cov(\nabla \log q_\theta (x_2))= I_1 + I_2
\end{align*}
\end{proof}
\begin{corollary}
If $x_i\simiid p_\theta$, $I(\theta) = Info(x_i)$, then $I_n(\theta) = nI(\theta)$. 
\end{corollary}
\paragraph{Information  Inequality}
We start with proving covariance ``lower bound". 

For any decision procedure $\delta: \cX\rightarrow \reals$ and any function $\psi: \cX \rightarrow \reals$, we have
\begin{align*}
\var(\delta) \geq \frac{\cov(\delta,\psi)^2}{\var(\psi)}
\end{align*}  
\begin{proof}
The proof uses Cauchy Schwarz. 
\begin{align*}
\cov(\delta,\psi) &= \E[(\delta-\E\delta)(\psi - \E\psi)]\\
				  &\leq \{\E[(\delta - \E\delta)^2]\}^{1/2}\{\E[(\psi - \E\psi)^2]\}^{1/2}\\
				  &= \{\var(\delta)\}^{1/2}\{\var(\psi)\}^{1/2}
\end{align*}
Therefore, we have 
\begin{align*}
\var(\delta) \geq \frac{\cov(\delta,\psi)^2}{\var(\psi)}
\end{align*}
\end{proof}
\begin{theorem}
(1 dimensional information inequality). Assume that $\E_\theta[\delta]= g(\theta)$ ids differentiable at $\theta$ and $P_\theta$ is regular enough that $\dot{P_\theta} = \frac{\partial}{\partial \theta}P_\theta$, $\int P_\theta d\mu = \frac{\partial }{\partial \theta}\int P_\theta d\mu = 0$, and $\int \delta(x) \dot{P_{\theta}(x)} d\mu = \frac{\partial }{\partial \theta}\E_{\theta}[\delta] = g'(\theta)$. Then $$\var_{\theta_0}(\delta)\geq\frac{(g'(\theta_0))^2}{I(\theta_0)} $$
\end{theorem}
Why do we care about this? Suppose that $\E_\theta[\delta] = 0$ and $g(\theta) = \theta$. Therefore, $g'(\theta) = 1$ and 
\begin{align*}
\E_\theta((\delta-\theta)^2) \geq \frac{1}{I(\theta)}
\end{align*}
In 1 dimension, any unbiased estimator has $MSE \geq \frac{1}{I(\theta)}$. However, this result is not true when our estimator is biased. 

\begin{proof}
Take $\psi(x) = \frac{\partial}{\partial \theta} \log P_\theta(x) = \frac{\partial}{\partial \theta} l_\theta(x) = \frac{\dot P_\theta(x)}{P_\theta(x)}$. 
By covariance inequality, 
\begin{align*}
\cov_\theta(\delta, \psi) &= \E_\theta[(\delta- g(\theta))(\psi - E[\psi])]\\
&=\E_\theta[\delta, \psi]\\
&=\E_\theta[\delta \frac{\dot P_\theta(x)}{P_\theta(x)}]\\
&=\int \delta(x) \dot P_\theta(x) \frac{P_\theta(x)}{P_\theta(x)}d\mu(x)\\
&= \frac{\partial}{\partial \theta} \int \delta P_\theta d\mu \\
&= \frac{\partial}{\partial \theta}\E_\theta[\delta] = g'(\theta)
\end{align*}
Therefore, we have 
\begin{align*}
\var(\delta) \geq \frac{\cov(\delta, \psi)^2}{\var(\psi)} = \frac{g'(\theta)}{I(\theta)}
\end{align*}
\end{proof}
\begin{remark}
This result is unsatisfying in two senses:
\begin{enumerate}
\item 
	Tied to mean square error (MSE)
\item 
	Requires unbiasedness
\end{enumerate}
We will cover a major theorem later in the class, which is better than this
result. Roughly speaking, we will show that
\begin{align*}
  \E_\theta[L(\sqrt n (\what{\theta}_n - \theta))]
  \geq \E[L(Z)], \ \ Z \sim \normal(0, I(\theta)^{-1}), \ \ \ \forall \what{\theta}_n
\end{align*}
and symmetric quasi-convex $L : \R^d \to \R_+$.  (In fact, this holds for
\emph{almost} all $\theta$, though not necessarily for all $\theta$.) More
precisely, the following result holds true.  Let $h \in \R^d$ and
$\theta_0 \in \Theta$ be arbitrary, where $\Theta \subset \R^d$ is open.  In
addition, assume that the distributions $P_\theta$ have log-likelihoods smooth
enough that the conditions of Theorem~\ref{theorem:asymptotic-normality} are
satisfied. Then for any sequence of estimators
$\what{\theta}_n : \mc{X}^n \to \Theta$ and any quasi-convex symmetric
loss $L : \R^d \to \R_+$,
\begin{equation*}
  \liminf_{c \to \infty}
  \liminf_n \sup_{\norm{h} \le c}
  \E_{\theta_0 + h / \sqrt{n}} \left[L(\sqrt{n} (\what{\theta}_n - (\theta_0 +
    h/\sqrt{n})
    ))\right] \ge \E[L(Z)],
  ~~ Z \sim \normal(0, I_{\theta_0}^{-1}).
\end{equation*}
That is, under perturbations of the true parameter $\theta_0$ by amounts
shrinking as $1 / \sqrt{n}$, we have a locally difficult estimation problem.
(Here $\E_\theta$ denotes expectation taken w.r.t.\
i.i.d.\ sampling under $P_\theta$.)


\end{remark}
\paragraph{Multidimensional Information Inequality}
We now generalize the result to $\theta \in \reals^d$. 
\begin{lemma}
Let $\delta: \cX \rightarrow \reals$, $\psi: \cX \rightarrow \reals^d$, $\E_\theta(\psi) = 0$. 
Define $\gamma = [\cov(\delta, \psi_j]_{j = 1}^d$, $C = \cov_\theta(\psi) = \E_\theta[\psi\psi^T]=0$. Then $$\var(\delta) \geq \gamma^TC^{-1}\gamma. $$
\end{lemma}
\begin{proof}
Let $v\in \reals^d$ be arbitrary. By 1 dimensional covariance inequality, 
\begin{align*}
\var(\delta) &\geq \frac{\cov(\delta, v^T\psi)^2}{\var(v^T\psi)}\\
&=\frac{(\gamma^Tv)^2}{v^TC v}= \gamma^T C^{-1} \gamma
\end{align*}
The last inequality uses the following fact: 
\begin{fact}
If $A > 0$, then 
\begin{align*}
\sup_v \frac{(v^Tu)^2}{v^TAv}= u^T A^{-1}u
\end{align*}
\end{fact}
\begin{inner-proof}
We first use Cauchy Schwartz. 
\begin{align*}
(v^Tu)^2 = (A^{1/2}v)^T(A^{-1/2}u)^2 \leq \|A^{1/2}v\|_2^2 \|A^{-1/2}u\|^2 = v^TAv u^TA^{-1}u
\end{align*}
Therefore, for all $v$, 
\begin{align*}
\frac{(v^Tu)^2}{v^TAv}\leq u^TA^{-1}u
\end{align*}
Now, we take $v = A^{-1}u$ to achieve this upper bound. 
\end{inner-proof}

\end{proof}
\begin{theorem}
Let $g(\theta) = \E_\theta[\delta]\in \reals^d$, with lots of regularity. 
Then we have \begin{align*}
\var_\theta(\delta) \geq \nabla g(\theta)^T I(\theta)^{-1}\nabla g(\theta)
\end{align*}
where $I(\theta) = \E_\theta[\nabla l_\theta \nabla l_\theta^T]$.
\end{theorem}
\begin{proof}
Let $\psi=\nabla l_\theta(x)$ in covariance lower bound. 
$\E_\theta[\psi]=0$, $\cov(\delta, \psi) = \E[\delta \nabla l_\theta] = \nabla \E_\theta[\delta] = \frac{\nabla \delta(\theta)}{\gamma}$. 
\end{proof}
\begin{corollary}
If $\hat \theta : x\rightarrow \Theta \in \reals^d$ is unbiased, 
\begin{align*}
\E_\theta[(\hat \theta - \theta) (\hat \theta - \theta)^T] \geq I(\theta)^{-1}
\end{align*}
\end{corollary}
\begin{proof}
Take $v \in \reals^d$, $\delta(x) = v^T \hat \theta$. Then 
\begin{align*}
\E(\delta) &= v^T\theta = g(\theta)\\
\Longrightarrow \nabla g(\theta) = v
\end{align*}
So 
\begin{align*}
\E[(v^T(\hat \theta- \theta))^2] &\geq v^T I(\theta)^{-1}v\\
&=\E[v^T(\hat \theta - \theta)(\hat \theta - \theta)^T v]
\end{align*}
\end{proof}
\end{document}
