\documentclass[11pt]{article}

% This package contains many of the macros you may need.
%
% A useful environment defined in it is the example environment, which you can
% invoke via \begin{example} and \end{example}.
\usepackage{scribe}

\begin{document}

\begin{center}
  \fcolorbox{black}{white}{
    \begin{minipage}{.9\columnwidth}
      \textbf{Stats 300b: Theory of Statistics}
      \hfill
      \textbf{Winter 2017} \\
      \vspace{-.2cm}
      % Instead of Lecture 1, January 10, put the actual lecture number and
      % date of the lecture here
      \begin{center}
        {\LARGE Lecture 3 -- January 17} \\
      \end{center}
      \emph{Lecturer: John Duchi}
      \hfill
      \emph{Scribe: John Cherian}  % You should put your name here if you scribe
    \end{minipage}
  }
\end{center}

\begin{center}
  \dbend ~~  \emph{\textbf{Warning:} these notes may contain factual errors}
\end{center}

\paragraph{Reading:} VDV Chapter 5.1-5.2, 5.5; TPE Chapter 2.5% Put the reading covered in the lecture here

%%%%%%%%%%%%%%%%% SCRIBE NOTES HERE %%%%%%%%%%%%%%%%%%%%
\paragraph{Outline of Lecture 2:}
\begin{enumerate}
\item Basic consistency and identifiability
\item Asymptotic Normality
\begin{enumerate}
\item Taylor expansions \& Fisher Information
\item Moment method (not covered)
\item Exponential Family models (not covered)
\end{enumerate}
\end{enumerate}

\paragraph{Recap:}
Last lecture, we discussed the Delta Method (aka Taylor expansions). The basic idea was as follows: \\
\begin{claim}
If $r_n(T_n - \theta) \cd T$, then if $\phi'(\theta)$ exists, $r_n(\phi(T_n) - \phi(\theta)) = r_n(\phi'(\theta)(T_n - \theta)) + o_p(r_n(T_n - \theta))$. Note that $r_n(T_n - \theta) \cp 0$, i.e. $o_p(1)$, since the sequence is uniformly tight. By applying the Continuous Mapping Theorem and Slutsky's Lemma, we conclude that $r_n(\phi(T_n) - \phi(\theta)) \cd \phi'(\theta)T$.
\end{claim}
\paragraph{Notation: }
\begin{definition}
Given distribution $P$ on $\mc{X}$, function $f: \mc{X} \to \R^d$, 
\begin{align*}
Pf \defeq \int fdP = \int_{\mc{X}} f(x) dP(x) = \E_{P}[f(x)]
\end{align*}
If $X_i$, $i = 1, \dots, n$, are observations, use $P_n$ to denote the empirical distribution:
\begin{align*}
P_nf \defeq \frac{1}{n} \sum_{i = 1}^n f(x_i) &&\text{for any function $f$}
\end{align*}
\end{definition}
\begin{example}
$P_n(A) = \frac{1}{n} \card{\{i \in [n] : x_i \in A\}}$
\end{example}
\paragraph{Setting:}
We have some model family $\{P_\theta\}_{\theta \in \Theta}$ of distributions on $\mc{X}$, where $\Theta \subseteq \R^d$. Also, assume all $P_\theta$ have density $p_\theta$ with respect to base measure $\mu$ on $\mc{X}$, i.e. $p_\theta = \frac{dP_\theta}{d\mu}$.
\paragraph{Idea:}
Our big idea for the day is that if we get enough observations from the same $P_\theta$, we should be able to:
\begin{enumerate}
\item Identify $\theta$
\item If $\ell_{\theta}(x) = \log{p_{\theta}(x)}$ is ``smooth'' enough, get explicit asymptotic normality results.
\end{enumerate}
\begin{definition}
\begin{align}
\nabla \ell_\theta(x) &\defeq \left [ \frac{\partial}{\partial \theta_j} \log{p_\theta(x)} \right]_{j = 1}^d \in \R^d \\
\nabla^2 \ell_\theta(x) &\defeq \left [ \frac{\partial^2}{\partial \theta_i \theta_j} \log{p_\theta(x)} \right ]_{i, j = 1}^d \in \R^{d \times d}
\end{align}
Note: $\dot{\ell}_\theta \equiv \nabla \ell_\theta(x)$ and $\ddot{\ell}_\theta \equiv \nabla^2 \ell_\theta(x)$.
\end{definition}
The gradient of the log-likelihood is often called the ``score function.'' We will use this term to refer to $\nabla \ell_\theta(x)$ throughout future lectures.
\paragraph{Estimating $\theta_0$:}
Observe $X_i \simiid P_{\theta_0}$ but $\theta_0$ is unknown. Our goal is to estimate $\theta_0$. 

A standard estimator is to choose $\hat{\theta}_n$ to maximize the ``likelihood,'' i.e. the probability of the data. 
\begin{align*}
\hat{\theta}_n \in \argmax_{\theta \in \Theta} P_n \ell_\theta(x)
\end{align*}

\begin{example}[Gaussian mean]
Let $X_i \simiid \normal(\theta, I)$ and $p_\theta(x) = \frac{1}{(2\pi)^{d/2}} e^{-\half \ltwo{x - \theta}^2} $. Then, $P_n\ell_n(\theta) = -\frac{1}{2n} \sum_{i = 1}^n \ltwo{x_i - \theta}^2 + \frac{d}{2} \log{(2\pi)}$. Note that this function is concave, so it has a unique global maximum. 

Since $\nabla P_n \ell_n(\theta) = -\frac{1}{n} \sum_{i = 1}^n (\theta - x_i) = 0$, $\hat{\theta}_n = \frac{1}{n} \sum_{i = 1}^n x_i$. And by the CLT, we know that $\sqrt{n}(\hat{\theta}_n - \theta_0) \cd \normal(0, I)$
\end{example}

\begin{example}[2-sided exponential]
Let $X_i \simiid Laplace(\theta_0)$ and $p_\theta(x) = \half e^{-|\theta - x|}$, i.e. $\log{p_\theta(x)} = -|\theta - x| - \log{2}$. Then, $\hat{\theta}_n \in \argmax_{\theta \in \Theta} -\frac{1}{n} \sum_{i = 1}^n |\theta - x_i|$, i.e. $\hat{\theta}_n = Median(x_1, \dots, x_n)$. 

Note that even though the likelihood function is not smooth (the absolute value function is not differentiable at 0), we are still able to derive asymptotic normality results for this estimator. These are, however, not presented in this lecture.
\end{example}

In general, there are two relevant questions to ask about Maximum Likelihood and other estimators of $\theta_0$:
\begin{enumerate}
\item Consistency: Does $\hat{\theta}_n \cp \theta_0$ as $n \to \infty$
\item What is the asymptotic distribution and the rate of convergence of $\hat{\theta}_n$ to $\theta_0$, i.e. for what $r_n \to \infty$, does $r_n(\hat{\theta}_n - \theta_0) \cd Z$ and what is $Z$?
\end{enumerate}

\paragraph{Consistency:}

\begin{definition}[Identifiability]

A model $\{P_\theta\}_{\theta \in \Theta}$ is \underline{identifiable} if $P_{\theta_1} \neq P_{\theta_2}$ for all $\theta_1, \theta_2 \in \Theta$ ($\theta_1 \neq \theta_2$). 

Equivalently, $\dkl{P_{\theta_1}}{P_{\theta_2}} > 0$ when $\theta_1 \neq \theta_2$. Recall that $\dkl{P_{\theta_1}}{P_{\theta_2}} = \displaystyle\int \log{ \frac{dP_{\theta_1}}{dP_{\theta_2}}} dP_{\theta_1}$.

Note that $P_{\theta_1} \neq P_{\theta_2}$ means that $\exists$ set $A \subseteq \mc{X}$ such that $P_{\theta_1} (A) \neq P_{\theta_2} (A)$.
\end{definition}

Now that we have established what both identifiability and consistency mean, we can prove a basic result regarding the finite consistency of the Maximum Likelihood estimator (MLE).

\begin{proposition}[Finite $\Theta$ consistency of MLE] 
Suppose $\{P_\theta \}_{\theta \in \Theta}$ is identifiable and $\card{\Theta} < \infty$. Then, if $\hat{\theta}_n \defeq \argmax_{\theta \in \Theta} P_n \ell_\theta(x)$, $\hat{\theta}_n \cp \theta_0$ when $X_i \simiid P_{\theta_0}$.
\end{proposition}
\begin{proof-of-proposition}
We know by the Strong Law of Large Numbers that $P_n \ell_\theta(x) \cas P_{\theta_0} \ell_\theta(x)$ when $x_i \simiid P_{\theta_0}$. 
\begin{align*}
P_{\theta_0} \ell_{\theta_0}(x) - P_{\theta_0} \ell_\theta(x) &= \E_{\theta_0} \left [ \log {\frac{p_{\theta_0}(x)}{p_{\theta}(x)}} \right] \\
&= \dkl{P_{\theta_0}}{P_{\theta}}
\end{align*}
We know that $\dkl{P_{\theta_0}}{P_{\theta}} > 0$ \underline{unless} $\theta = \theta_0$. So, for sufficiently large $n$ and finite $\Theta$, we have that:
\begin{align*}
P_n \ell_{\theta_0} (x) - P_n \ell_\theta (x) > 0 &&\text{$\forall \theta \neq \theta_0$}
\end{align*}
Therefore, $\hat{\theta}_n = \theta_0$ ``eventually.''
\end{proof-of-proposition}
\begin{remark}
The above result can fail for $\Theta$ infinite even if $\Theta$ is countable.
\end{remark}

One sufficient condition often used for consistency results is a \underline{uniform law}, i.e. for $x_i \simiid P$, we have that $\sup_{\theta \in \Theta} |P_n \ell_\theta - P \ell_\theta| \cp 0$.

\begin{example}[Uniform laws]
Suppose $\Theta = \R^d, x_i \simiid P$. Consider $P_n \< \theta, x \> = \frac{1}{n} \sum_{i = 1}^n \< \theta, x_i \>$. Then, for each $\theta \in \Theta$, $P_n \<\theta, x\> \cas \< \theta, \E[x] \>$. 

Does the uniform law, i.e. $\sup_{\theta \in \Theta} | \< \theta, P_n x \> - \< \theta, P x\>| \to 0$, hold? If $\cov{x} \neq 0$, no. However, if $\Theta$ is compact, we will be able to prove this result.

Alternatively, $\hat{\theta}_n = \argmax_{\theta \in \Theta} P_n \< x, \theta \>$ does satisfy $\< \hat{\theta}_n , Px \> \to \sup_{\theta \in \Theta} \< \theta, Px \>$.
\end{example}

Now, that we have established some basic definitions and results regarding the consistency of estimators, we turn our attention to understanding their asymptotic behavior.
\paragraph{Asymptotic Normality via Taylor Expansions:}
\begin{definition}[Operator norm]
$\opnorm{A} \defeq \sup_{\ltwo{u} \leq 1} \ltwo{Au}$. \\
Note: $A \in \R^{k \times d}, u \in \R^d$ and $\ltwo{Ax} \leq \opnorm{A} \ltwo{x}$.
\end{definition}
Before we do anything, we have to make several assumptions. 
\begin{enumerate}
\item We have a ``nice, smooth'' model, i.e. the Hessian is Lipschitz-continuous. To be rigorous, the following must hold:
\begin{align*}
\opnorm{\nabla^2 \ell_{\theta_1}(x) - \nabla^2 \ell_{\theta_2}(x)} \leq M(x) \ltwo{\theta_1 - \theta_2} &&\text{$\E_\theta[M^2(x)] < \infty$}
\end{align*}
\item The MLE, $\hat{\theta}_n \in \argmax_{\theta \in \Theta} P_n \ell_\theta(x)$, is consistent, i.e. $\hat{\theta}_n \cp \theta_0$ under $P_{\theta_0}$.
\item $\nabla P_n \ell_{\hat{\theta}_n} = 0$.
\item $\Theta$ is a convex set.
\end{enumerate}
\begin{theorem}
Let $x_i \simiid P_{\theta_0}$ and assume the conditions stated above. Then, $\sqrt{n}(\hat{\theta}_n - \theta_0) \cd \normal (0, (P_{\theta_0} \nabla^2 \ell_{\theta_0})^{-1} P_{\theta_0} \nabla \ell_{\theta_0} \nabla \ell_{\theta_0}^T (P_{\theta_0} \nabla^2 \ell_{\theta_0})^{-1})$.
\end{theorem}
\begin{remark}
We define the \underline{Fisher Information} as $I_\theta \defeq \E_\theta [ \nabla \ell_\theta(x) \nabla \ell_\theta(x)^T ] = \cov_\theta{\nabla \ell_\theta}$. 

The final equality holds because $\E_\theta[\nabla \ell_\theta(x)] = 0$ ($\theta$ maximizes $\E_\theta[\ell_\theta(x)]$).

To show this, assume that we can swap $\nabla, \E$. Then, $\nabla \ell_\theta(x) = \nabla \log{p_\theta(x)} = \frac{\nabla p_\theta(x)}{p_\theta(x)}$. Using that result, we see that:
\begin{align*}
\E_\theta[\nabla \ell_\theta] &= \E\left [ \frac{\nabla p_\theta}{p_\theta} \right] \\
&= \int \frac{\nabla p_\theta}{p_\theta} p_\theta d\mu = \int \nabla p_\theta d\mu \\
&= \nabla \int p_\theta d\mu = \nabla (1) = 0.
\end{align*}
Similarly, given that $\nabla^2 \ell_\theta = \nabla \left ( \frac{\nabla p_\theta}{p_\theta} \right) = \frac{\nabla^2 p_\theta}{p_\theta} - \frac{\nabla p_\theta \nabla p_\theta^T}{p^2_\theta}$:
\begin{align*}
\E_\theta\left [ \frac{\nabla^2 p_\theta}{p_\theta} \right] &= \int \frac{\nabla^2 p_\theta}{p_\theta} p_\theta d\mu \\
&= \int \nabla^2 p_\theta d\mu = \nabla^2 \int p_\theta d\mu = 0
\end{align*}
As a result:
\begin{align*}
\E_\theta[\nabla^2 \ell_\theta] &= - \E_\theta \left [ \left (\frac{\nabla p_\theta}{p_\theta} \right) \left ( \frac{\nabla p_\theta}{p_\theta} \right)^T \right] \\
&= - \cov_\theta (\nabla \ell_\theta(x)) = -I_\theta
\end{align*}
\end{remark}

Using what we have shown about the Fisher information, we now have a more compact representation of the asymptotic distribution described in the Theorem above.
\begin{align}
\sqrt{n} (\hat{\theta}_n  -\theta_0) \cd \normal (0, I_{\theta_0}^{-1} I_{\theta_0} I_{\theta_0}^{-1}) = \normal (0, I_{\theta_0}^{-1})
\end{align}
Consider $I_\theta = - \nabla^2 \E[\ell_\theta(x)]$. If the magnitude of the second derivative is ``large,'' that implies that the log-likelihood is steep around the global maximum (making it ``easy'' to find). Alternatively, if the magnitude of $- \nabla^2 \E[\ell_\theta(x)]$ is ``small,'' we do not have sufficient curvature to find the optimal $\theta$.
\end{document}
