\documentclass[11pt]{article}

% This package contains many of the macros you may need.
%
% A useful environment defined in it is the example environment, which you can
% invoke via \begin{example} and \end{example}.
\usepackage{scribe}

\begin{document}

\begin{center}
  \fcolorbox{black}{white}{
    \begin{minipage}{.9\columnwidth}
      \textbf{Stats 300b: Theory of Statistics}
      \hfill
      \textbf{Winter 2017} \\
      \vspace{-.2cm}
      % Instead of Lecture 1, January 10, put the actual lecture number and
      % date of the lecture here
      \begin{center}
        {\LARGE Lecture 9 -- February 7} \\
      \end{center}
      \emph{Lecturer: John Duchi}
      \hfill
      \emph{Scribe: Dylan Cable}  % You should put your name here if you scribe
    \end{minipage}
  }
\end{center}

\begin{center}
  \dbend ~~  \emph{\textbf{Warning:} these notes may contain factual errors}
\end{center}

\paragraph{Reading: Chapter 7.7 of Lehmann's Elements of Large Sample Theory} % Put the reading covered in the lecture here

%%%%%%%%%%%%%%%%% SCRIBE NOTES HERE %%%%%%%%%%%%%%%%%%%%
$\\$
$\\$
Today we are going to finish up hypothesis and confidence tests.  We will return to them when we talk about optimality.  Then, we will talk about uniform laws of large numbers - how you give law of large numbers uniformly over a function class.

$\\$

$\bold{Outline: } \\$ 
Parametric Big 3 tests (Chapter 7.7 of Lehmann's Elements of Large Sample Theory)

\begin{itemize}
  \item Likelihood Ratio Test
  \item Wald Test
  \item Rao (score tests)
\end{itemize}

Begin Uniform Laws of Large Numbers (ULLNs)

\begin{itemize}
  \item Concentration inequalities
  \item Sub-Gaussianity
\end{itemize}

\textbf{Recap: Generalized Likelihood Ratio Test:}
Let $(x_1,\dots,x_n) = x$, $L_n(x;\theta):= \sum_{i=1}^nl_\theta(x_i), l_\theta(x) = \log p_\theta(x)$

Let $\hat{\theta}_n = \argmax_\theta L_n(x;\theta)$ and assume usual asymptotic normality conditions hold (twice differentiable, Lipschitz-continuous hessian).  Then $\delta_n = L_n(x;\hat{\theta}_n) - L_n(x;\theta)$ satisfies

$$2\delta_n \underset{P_{\theta_0}}{\rightarrow} \chi_d^2, \theta_0 \in \mathbb{R}^d$$

Definition: Let $T_n$ be a sequence of tests for some model $\{P_\theta\}_{\theta \in \Theta}$, let 
$$H_0: \theta \in \Theta_0 \subset \Theta$$

Then $T_n$ is \underline{asymptotically level $\alpha$} if 

$$\underset{n\rightarrow\infty}{\lim}\underset{\theta \in \Theta_0}{\sup}P_\theta(T_n \text{ rejects } H_0) \leq \alpha$$

Example: Generalized Likelihood Ratio Test is defined by reject if $\delta_n \geq u_{d,\alpha}^2/2$, where $u_{d,\alpha}^2$ is defined so that $\mathbb{P}(\chi_d^2 \geq u_{d,\alpha}^2) = \alpha$.  This test is asymptotically level $\alpha$.

\underline{Wald Tests}
Define \underline{Wald confidence ellipsoid} for model $\{P_\theta\}$ with Fisher information $I_\theta$ by
$$C_{n,\gamma} = \{\theta \in \Theta : (\theta - \hat{\theta}_n)^TI_{\hat{\theta}_n}(\theta - \hat{\theta}_n) \leq \gamma/n\}$$

where $ \hat{\theta}_n$ is Maximum Likelihood Estimator.  We saw last time that 

$$ n(\theta - \hat{\theta}_n)^TI_{\hat{\theta}_n}(\theta - \hat{\theta}_n) \underset{P_{\theta_0}}{\rightarrow} \chi_d^2$$

Wald test of $H_0: \theta = \theta_0$ against $\theta \neq \theta_0$:
$\\$
Acceptance: $\hat{\theta}_n \in \{\theta \in \Theta : (\theta - {\theta}_0)^TI_{\hat{\theta}_n}(\theta - {\theta}_0 \leq u_{d,\alpha}^2/n\} := A_n$ 

This is asymptotically level $\alpha$. Why?

$$ n(\hat{\theta}_n - \theta_0)^TI_{\hat{\theta}_n}(\hat{\theta}_n - \theta_0) \underset{P_{\theta_0}}{\rightarrow} \chi_d^2$$

$$P_{\theta_0} ( \hat{\theta}_n \notin A_n) = P_{\theta_0} ((\hat{\theta}_n - \theta_0)^TI_{\hat{\theta}_n}(\hat{\theta}_n - \theta_0)  > u_{d,\alpha}^2/n) \rightarrow \mathbb{P}(\chi_d^2 \geq u_{d,\alpha}^2) = \alpha $$

\underline{Remark:} If $H_0 = \theta = \theta_0$ is a point null, we can replace $I_{\hat{\theta}_n}$ with $I_{\theta_0}$.

What about composite nulls (nuisance parameters)?

\underline{Example:}

$X_i \underset{i.i.d.}{\sim} N(\theta,\sigma^2), \sigma^2$ is unknown.

$H_0: \theta = 0$, but $\sigma^2$ is unspecified.

To deal with this, let $\Sigma (\theta) = I^{-1}(\theta)$ so that $\sqrt{n}(\hat{\theta}_n - \theta) \overset{d}{\underset{P_{\theta_0}}{\rightarrow}} N(0,\Sigma(\theta))$

For vector $v \in \mathbb{R}^d$, let $[v]_{1:k} = \begin{bmatrix}v_1 \\ \vdots \\ v_k \end{bmatrix} \in \mathbb{R}^k$

For $\Sigma = (e_{ij})_{i,j=1}^d \in \mathbb{R}^{d\times d}$, let $\Sigma^{(k)}$ be the leading k-by-k principal submatrix $(e_{ij})_{i,j=1}^k$

Then $\sqrt{n}([\hat{\theta_n}]_{1:k} - [\theta]_{1:k}) \rightarrow N(0,\Sigma^{(k)}(\theta))$

\underline{Fact}: if $A =  \begin{bmatrix}A_{11} A_{12} \\  A_{21} A_{22} \end{bmatrix}, M = A^{-1}$, then $M_{11} = (A_{11} - A_{12}A_{22}^{-1}A_{21})^{-1}$ (exercise).

Then $\Sigma^{(k)}(\theta) = ((I(\theta))^{-1})^{k} = (I^k(\theta) - I_{12}(\theta)I_{22}(\theta)^{-1}I_{21}(\theta))^{-1}\\$
Then $\Sigma^{(k)}(\theta) \succeq (I^{(k)}(\theta))^{-1}$ and equal if and only if $I_{12}(\theta) = 0$ (i.e. $[\hat{\theta_n}]_{1:k}$ and $[\hat{\theta_n}]_{k+1:d}$ are asymptotically independent.)

$$n([\hat{\theta_n}]_{1:k} - [\theta_0]_{1:k})^{T} \Sigma^{(k)}(\hat{\theta_n})^{-1} ([\hat{\theta_n}]_{1:k} - [\theta_0]_{1:k}) \underset{\theta_0}{\rightarrow} \chi_k^2$$
Let $H_0: \theta_1 = \theta_1^0,\dots,\theta_k=\theta_k^0$, and $\theta_{k+1},\dots,\theta_d$ are unspecified

$$\hat{\theta_n} \in \{\theta : ([\theta]_{1:k} - [\theta^0]_{1:k})^{T} \Sigma^{(k)}(\hat{\theta_n})^{-1} ([\theta]_{1:k} - [\theta^0]_{1:k}) \leq u^2_{k,\alpha}/n \},  \mathbb{P}(\chi_k^2 \geq u_{k,\alpha}^2) = \alpha$$

Then this test is asymptotically level $\alpha$.  We must use $\Sigma(\hat{\theta_n}) = I(\hat{\theta_n})^{-1}$ to get a consistent estimator of $I(\theta)$ under $H$.

\underline{Example}: Gaussian mean, unknown covariance. $N(\theta,\Sigma), \theta \in \mathbb{R}^d,\Sigma \succ 0$

$\bar{x}_n = \frac{1}{n}\sum_{i=1}^nx_i$, $\hat{\Sigma} = \frac{1}{n}\sum_{i=1}^n(x_i-\bar{x}_n)(x_i-\bar{x}_n)^{T}$

The test is reject if $\{ \bar{x}_n^{T} \hat{\Sigma}^{-1} \bar{x}_n \geq u_{d,\alpha}^2)/n \}$
This is because the fisher information of Gaussian is $\hat{\Sigma}^{-1}$.

\underline{Rao Score Test:}
Why the Rao Score Test? The MLE is a bit difficult to compute.

Suppose I have a point null $H_0: \theta = \theta_0$

What is the limit of $\sqrt{n} P_n \nabla l_{\theta_0}$ under $H_0$?

$$\sqrt{n} P_n \nabla l_{\theta_0} \rightarrow N(0,I_{\theta_0})$$

$$n(P_n \nabla l_{\theta_0})^TI_{\theta_0}^{-1}(P_n \nabla l_{\theta_0}) \underset{P_{\theta_0}}{\rightarrow} \chi_d^2$$

Define: Rao test of $H_0: \theta = \theta_0$ vs $\theta \neq \theta_0$: Reject if $(P_n \nabla l_{\theta_0})^TI_{\theta_0}^{-1}(P_n \nabla l_{\theta_0}) \geq u^2_{d,\alpha}/n$.  This is asymptotically level $\alpha$.

\underline{Remarks:} (1) Analogues for composite nulls.  (2) Strong connections to optimal testing (we will see later).

\underline{Uniform Laws of Large Numbers:} (ULLN)
Basic questions: Given collection $\mathcal{F}$ of functions $\subset \{f: X \rightarrow \mathbb{R}\}$, when do we have $\sup_{f \in \mathcal{F}}|P_nf - Pf| \underset{p}{\rightarrow} 0$?

Why should we care? M-estimators: Let $m_\theta:X\rightarrow \mathbb{R}$ be defined for $\theta \in \Theta$.  The associated M-estimator is 

$$\hat{\theta}_n = \argmax_{\theta \in \Theta}P_nm_\theta(x)$$

When does $\hat{\theta}_n \rightarrow \argmax_{\theta \in \Theta}Pm_\theta(x)$

Let $M(\theta) := P{m_\theta}(x)$ be the population objective.
$M_n(\theta) := P_n{m_\theta}(x)$. 

When does $M(\hat{\theta}_n) \underset{p}{\rightarrow} \sup_{\theta \in \Theta}M(\theta) $?

Suppose we had a ULLN: $\sup_{\theta \in \Theta}|M_n(\theta) - M(\theta)| \underset{p}{\rightarrow} 0$

Let $\hat{\theta}_n = \argmax_{\theta \in \Theta}M_n(\theta)$

$M_n(\hat{\theta}_n) = \frac{1}{n}\sum_{i=1}^nm_{\hat{\theta}_n}(x_i)$ is not an i.i.d sum because $\hat{\theta}_n$ depends on $(x_i)$.

But $M(\hat{\theta}_n) - M(\theta^*) = (M(\hat{\theta}_n) - M_n(\hat{\theta}_n)) + (M_n(\hat{\theta}_n) - M_n(\theta^*))+(M_n(\theta^*) - M(\theta^*))$

$\geq \sup_{\theta \in \Theta}|M_n(\theta) - M(\theta)| + 0 + o_p(1)$

Applying our ULLN, we conclude $M(\hat{\theta}_n) \underset{p}{\rightarrow} M(\theta^*)$

\end{document}

