\documentclass[11pt]{article}

% This package contains many of the macros you may need.
%
% A useful environment defined in it is the example environment, which you can
% invoke via \begin{example} and \end{example}.
\usepackage{scribe}

\begin{document}
\setlength\parindent{0pt}
\begin{center}
  \fcolorbox{black}{white}{
    \begin{minipage}{.9\columnwidth}
      \textbf{Stats 300b: Theory of Statistics}
      \hfill
      \textbf{Winter 2017} \\
      \vspace{-.2cm}
      \begin{center}
        {\LARGE Lecture 8 -- February 2} \\
      \end{center}
      \emph{Lecturer: John Duchi}
      \hfill
      \emph{Scribe: Dylan Greaves}  % You should put your name here if you scribe
    \end{minipage}
  }
\end{center}

\begin{center}
  \dbend ~~  \emph{\textbf{Warning:} these notes may contain factual errors}
\end{center}

\paragraph{Reading:} % Put the reading covered in the lecture here


%%%%%%%%%%%%%%%%% SCRIBE NOTES HERE %%%%%%%%%%%%%%%%%%%%

\paragraph{Outline:}
\begin{itemize}
    \item Finish U-statistics
    \item Testing and Confidence Intervals
    \item Duality between Testing and Confidence
    \item (Generalized) Likelihood Ratio Tests
\end{itemize}


\paragraph{Recap:}
Last lecture, we proved the following two results:
\begin{claim}
Let $U_n =\binom{n}{r}^{-1}\sum_{|\beta|=r} h(X_\beta),$ and $h_c(x_1,\hdots,x_c)=\E[h(x_1,\hdots,x_c,X_{c+1},\hdots, X_r)],$ then
$$\var (U_n)=\frac{r^2}{n}\zeta_1+O(n^{-2}),$$
where $\zeta_1=\var (h_1)$.
\end{claim}
\begin{claim}
    If $\mathcal{S}$ is a linear subspace and $\hat{S}_n$ is the projection of $T_n$ on $\mathcal{S}$, then 
    $$\frac{\var(\hat{S}_n)}{\var(T_n)}\rightarrow 1 \implies \frac{T_n-\E T_n}{\sqrt{\var(T_n)}}-\frac{\hat{S}_n-\E \hat{S}_n}{\sqrt{\var(\hat{S}_n)}}\xrightarrow{P} 0.$$
\end{claim}

We will combine these two ideas to show the asymptotic normaility of U-statistics.
\section{Asymptotic Normality of U-statistics}
(Haj\'{e}k) The main idea is to use projections onto sets of the form 
$$\mathcal{S}_n=\{\sum_{i=1}^n g_i(X_i):g_i(X_i)\in L_2(P)\}.$$

\begin{theorem}
    Let $h$ be a symmetric kernel (function) of order $r$ and let $\E h^2 <\infty$, $U_n$ be the associated U-statistic, then
    $$\sqrt{n}(U_n-\theta)\xrightarrow{d}\normal(0,r^2\zeta_1),$$
    where $\theta=\E U_n = \E h(X_1,\hdots,X_n)$.
\end{theorem}
\begin{proof}
    Let $\hat{U}_n$ be defined as $\hat{U}_n=\sum_{i=1}^n\E[U_n-\theta|X_i]$, then $\hat{U}_n$ is the projection of $U_n-\theta$ onto $\mathcal{S}_n$. Let's compare the variances and expectations.\\

    Let $\beta\subseteq[n]$, $|\beta|=r$, then
    $$\E[h(X_\beta)-\theta|X_i]=\begin{cases} 
        0 & i\not\in \beta\\
        h_1(X_i) & i\in\beta
    \end{cases}.
    $$

    Then \begin{align*}
        \E[U_n-\theta|X_i] &= \binom{n}{r}^{-1}\sum_{|\beta|=r}\E[h(X_\beta)-\theta|X_i=x]\\
        &=\binom{n}{r}^{-1}\sum_{|\beta|=r,i\in\beta}h_1(X_i)\\
        &=\binom{n}{r}^{-1}\binom{n-1}{r-1}h_1(X_i)=\frac{r}{n}h_1(X_i)
    \end{align*}

    It follows that $\hat{U}_n=\frac{r}{n}\sum_{i=1}^nh_1(X_i)$ and certainly $\sqrt{n}(\hat{U}_n-\theta)\xrightarrow{d}\normal(0,r^2\zeta_1)$.\\

    Now apply ratio of variance condition (Claim 2), since
    \begin{align*}
        \var(U_n) &= \frac{r^2}{n}\zeta_1+O(n^{-2})\\
        \var(\hat{U}_n) &= \frac{r^2}{n}\zeta_1
     \end{align*}
    
    we have $\frac{\var(U_n)}{\var(\hat{U}_n)}\rightarrow 1$ as $n\rightarrow\infty$, from which we conclude $U_n$ and $\hat{U}_n$ have the same asymptotic behavior.
\end{proof}

\section{Testing and Confidence Intervals}
We've seen a number of scenarios where
$$\sqrt{n}(\hat{\theta}_n-\theta_0)\xrightarrow{d}\normal(0,\Sigma).$$
Suppose we would like to make the following claim about the population parameter $\theta_0$: "With reasonably high confidence, $\theta_0\in \mathcal{C}_n$, where $\mathcal{C}_n\subseteq\R^d$ is a set."\\

\begin{example}
    Suppose $\sqrt{n}(\hat{\theta}_n-\theta_0)\xrightarrow[\theta_0]{d}\normal(0,I_{\theta_0}^{-1})$. Say $I_\theta$ is continuous in $\theta$ and $I_\theta$ is invertible. Let
    $$C_{n,\gamma}:= \{\theta\in\R^d:(\theta-\hat{\theta}_n)^T I_{\hat{\theta}_n}(\theta-\hat{\theta}_n)\leq \frac{\gamma}{n}\}.$$

    For $\theta=\theta_0$, we have
    \begin{align*}
        n(\theta_0-\hat{\theta}_n)^T I_{\hat{\theta}_n}(\theta_0-\hat{\theta}_n) &= (\sqrt{n}(\hat{\theta}_n-\theta_0))^T(I_{\theta_0}+o_P(1))(\sqrt{n}(\hat{\theta}_n-\theta_0)\\
        &= (\underbrace{\sqrt{n}(\hat{\theta}_n-\theta_0)}_{\xrightarrow{d} \normal(0,I_{\theta_0}^{-1})})^TI_{\theta_0}(\sqrt{n}(\hat{\theta}_n-\theta_0))+o_P(1)\\
        &\xrightarrow{d} Z^T I_{\theta_0}Z & Z\sim\normal(0,I_{\theta_0}^{-1})\\
        &\stackrel{d}{\equiv} \norm{W}_2^2\stackrel{d}{\equiv}\chi^2_d & W\sim \normal(0,I_{d\times d})
    \end{align*}
    Then 
    \begin{align*}
        \P_{\theta_0}(\theta_0\in  C_{n,\gamma})&=\P_{\theta_0}((\theta_0-\hat{\theta}_n)^TI_{\hat{\theta}_n}(\theta_n-\hat{\theta}_n)\leq \frac{\gamma}{n})\\
        &\rightarrow\P(\norm{W}_2^2\leq \gamma) & W\sim\normal(0,I_{d\times d})\\
        &= \P(\chi^2_d\leq \gamma).
    \end{align*}

$C_{n,\gamma}$ is pivotal, since it doesn't depend on the parameter $\theta_0$. 
If for some level $\alpha <1$ you want that $P_{\theta_0}(\theta_0\in C_{n,\gamma})\rightarrow \alpha$, take $C_n= C_{n,\gamma}$, where $\gamma$ is chosen such that $P(\chi^2_d\leq \gamma)=\alpha$.
\end{example}

\section{Dual Problem to Confidence Sets}
The typical approach to hypothesis testing is the following: Can we reject some type of null hypothesis, that is supposingly conjecture $H_0:P_{\theta_0}$? Can we get results like 
$$P_{\theta_0}(\text{data at least as ``extreme" as what we got})\leq \alpha?$$

It's questionable whether this is even a reasonable thing to do, since this is a ill-formed definition -- ``extreme" is vague. One might also take philosophical issue with this approach, since the only conclusions that result from it are negative statements -- ``this null hypothesis doesn't explain the world." While this may be troubling, it's worthwhile to note that this is also nature of the scientific method: scientific hypotheses are never proven ``true," prevailing hypotheses are only held until they are falsified by new observations (e.g. Michelson-Morley experiment (1887) and the \ae ther drag hypothesis).

\begin{definition}[p-value] Let $H_0:\{P_\theta:\theta\in\Theta_0\}$. The p-value associated with a sample $X_1,\hdots, X_n$ is defined to be 
    $$\sup_{\theta\in\Theta_0}P_\theta(\text{data as extreme as $X_1,\hdots,X_n$ observed})$$
\end{definition}

\begin{example}
    Let $H_0:X_i\simiid \normal(0,1)$. The standard p-value is given by
    $$P_0(|\bar{Z}|>|\hat{\theta}|),$$
    where $\hat{\theta}_n=\frac{1}{n}\sum_{i=1}^nX_i.$
\end{example}

How can we understand these and develop a few tests with reasonable properties?

\section{Generalized Likelihood Ratio Tests}

Recall the classic Neyman-Pearson setup with simple null and alternative:
\begin{align*}
    H_0 &: p_0\\
    H_1 &: p_1
\end{align*}
The test that is the ``best" (maximizes power at all levels) is the likelihood ratio test, where if 
$L_1(x) = \log dP_1(x)$, $L_0(x)=\log dP_0(x)$, and $T(x)=L_1(x)-L_0(x)=\log \frac{dP_1}{dP_0}(x)$, the most powerful test  is given by
$$\begin{cases}
    \text{accept $H_1$/reject $H_0$} & T>t\\
    \text{accept $H_0$/reject $H_1$} & T<t\\
    \text{balance} & T=t
\end{cases}
$$
for some $t$.

\subsection{Generalized LRT}

We now consider a more general scenario with composite null and alternative. Suppose
\begin{align*}
    H_0 &: \theta\in \Theta_0\\
    H_1 &: \theta\in \Theta
\end{align*}
where usually $\Theta_0\subseteq \Theta$.
Define
$$T(x)=\log \frac{\sup_{\theta\in\Theta}p(x,\theta)}{\sup_{\theta\in\Theta_0}p(x,\theta)}=\frac{p(x,\hat{\theta}_{\text{MLE}})}{\sup_{\theta\in\Theta_0}p(x,\theta)}.$$
The Generalized LRT rejects $H_0$ if $T(x)>t$.\\

Suppose $\{P_\theta\}_{\theta\in\Theta}$ is nice enough that the MLE is asymptotically normal, $$I_{\theta_0}=\E \nabla \ell_{\theta_0}\ell_{\theta_0}^T=-\E\nabla^2\ell_{\theta_0},$$ and
$$\opnorm{\nabla^2\ell_\theta(x)-\nabla^2\ell_{\theta'}(x)}\leq M(x)\norm{\theta-\theta'},$$ where $\E_{\theta}M^2(X)<\infty$. Then we have the following asymptotic result:
\begin{proposition}[Wilk's Theorem]
    Let $\Theta_0=\{\theta_0\}$ be a point null, $\Theta=\R^d$. Let $L_n(x,\theta)=\sum_{i=1}^n\ell_\theta(x_i)=\sum_{i=1}^n \log p_\theta(x_i)$ and $T_n(x)=L_n(x,\hat{\theta}_{\text{MLE}})-L_n(x,\theta_0)$. Then
    $$2T_n(X)\xrightarrow[\theta_0]{d}\chi^2_d,$$
    where $X=(X_1,\hdots,X_n)$ and $X_i\simiid P_{\theta_0}$.
\end{proposition}
\begin{proof}
    Let $\hat{\theta}_n=\argmax_{\theta\in\Theta}L_n(X,\theta)=\hat{\theta}_{\text{MLE}}$. 

    Under $H_0$,
    $\hat{\theta}_{\text{MLE}}-\theta_0\xrightarrow[\theta_0]{P}0$ and $\sqrt{n}(\hat{\theta}_{\text{MLE}}-\theta_0)\cd\normal(0,I_{\theta_0}^{-1})$.

By Taylor's Theorem, we have
    $$0=\nabla L_n(X,\hat{\theta}_n)=\nabla L_n(X,\theta_0)+\nabla^2 L_n(X,\theta_0)(\hat{\theta}_n-\theta_0)+\sum_{i=1}^n\text{Err}_i^{(1)}(\hat{\theta}-\theta),$$
    where $\opnorm{\text{Err}_i^{(1)}}\leq M(X_i)\norm{\hat{\theta}_n-\theta_0}.$

Similarly,
    $$L_n(X,\hat{\theta}_n)=L_n(X,\theta_0)+\nabla L_n(X,\theta_0)^T(\hat{\theta}_n-\theta_0)+\frac{1}{2}(\hat{\theta}_n-\theta_0)^T\nabla^2 L_n(X,\theta_0)(\hat{\theta}_n-\theta_0)+\sum_{i=1}^n (\hat{\theta}_n-\theta_0)^T\text{Err}_i^{(2)}(\hat{\theta}_n-\theta_0)$$
where $\opnorm{\text{Err}_i^{(2)}}\leq M(X_i)\norm{\hat{\theta}_n-\theta_0}.$

    Substituting the first equation into the second, and letting $\text{Err}_i = \text{Err}_i^{(2)}-\text{Err}_i^{(1)}$, we have
    $$T(X)=L_n(X,\hat{\theta}_n)-L_n(X,\theta_0)=-\frac{1}{2}(\hat{\theta}_n-\theta_0)^T\nabla^2 L_n(X,\theta_0)(\hat{\theta}_n-\theta_0)+\sum_{i=1}^n(\hat{\theta}_n-\theta_0)^T\text{Err}_i(\hat{\theta}_n-\theta_0).$$
Since
    $\frac{1}{n}\sum_{i=1}^n\nabla^2\ell_{\theta_0}(X_i)\xrightarrow{P}-I_{\theta_0}$, $\sum_{i=1}^n\text{Err}_i\xrightarrow{P}0$, and $\sqrt{n}(\hat{\theta}_n-\theta)\xrightarrow[H_0]{d}\normal(0,I_{\theta_0}^{-1})$, it follows that
    $$2T(X)=\sqrt{n}(\hat{\theta}_n-\theta_0)^TI_{\theta_0}(\hat{\theta}_n-\theta_0)+o_P(1)\xrightarrow{d}\chi^2_d.$$
\end{proof}

\end{document}
