\documentclass[11pt]{article}

% This package contains many of the macros you may need.
%
% A useful environment defined in it is the example environment, which you can
% invoke via \begin{example} and \end{example}.
\usepackage{scribe}

\begin{document}

\begin{center}
  \fcolorbox{black}{white}{
    \begin{minipage}{.9\columnwidth}
      \textbf{Stats 300b: Theory of Statistics}
      \hfill
      \textbf{Winter 2017} \\
      \vspace{-.2cm}
      % Instead of Lecture 1, January 10, put the actual lecture number and
      % date of the lecture here
      \begin{center}
        {\LARGE Lecture 20 -- March 16} \\
      \end{center}
      \emph{Lecturer: John Duchi}
      \hfill
      \emph{Scribe: Jonathan Lacotte}  % You should put your name here if you scribe
    \end{minipage}
  }
\end{center}

\begin{center}
  \dbend ~~  \emph{\textbf{Warning:} these notes may contain factual errors}
\end{center}

\paragraph{Reading:} 

\paragraph{Outline}
\begin{itemize}
\item limiting Gaussian experiments
\item local asymptotic minimax theorem
\end{itemize}

\section{Limiting Gaussianity}
{\bf Definition 1.1. }  A collection $\{P_{\theta, n}\}_{\theta \in \Theta,\,n \in \mathbb{N}}$ is locally asymptotically normal (LAN) with precision/information $K_{\theta} \in \mathbb{R}^{d \times d}$ if there exists $\Delta_n \in \mathbb{R}^d$ such that: 
$$\log\left(\frac{dP_{\theta+\frac{h}{\sqrt{n}},n}}{dP_{\theta, n}}(X^n)\right) = h^T \Delta_n(X^n) - \frac{1}{2}h^TK_{\theta}h+o_{P_{\theta, n}}(||h||)$$
where $\Delta_n(X^n) \xrightarrow[P_{\theta, n}]{d} \mathcal{N}(0, K_{\theta})$.\\
\\
Le Cam's third lemma implies that, with $Z_n=K_{\theta}^{-1}\Delta_n$, $Z_n \xrightarrow[P_{\theta+\frac{h}{\sqrt{n}},n}]{d}\mathcal{N}(h, K_{\theta}^{-1})$\\
\\
The goal is to make rigorous that in LAN families, estimating $\theta$ is same (in limit) as estimating the mean from a Gaussian location family.\\
\\
Throughout, we assume that $\theta_0 = 0$ (wlog).\\
\\
{\bf Lemma 1.1. } Let $Z_n = K^{-1}\Delta_n$ (in LAN family). Then, $(Z_n)$ is uniformly tight under $(P_{\frac{h}{\sqrt{n}},n})$ whenever $||h|| \le C < +\infty$.\\
Moreover, if we define $dQ_{h,n}(z)=\exp\left(-\frac{1}{2}(z-h)^TK(z-h) - z^TKz\right)dP_{\theta, n}(z)$ then we get for all $c, b < + \infty$:
$$\limsup_{n \infty} \sup_{||h|| \le c} \int \mathbf{1}_{\{||z|| \le b\}} |dQ_{h, n}(z) - dP_{\frac{h}{\sqrt{n}},n}(z)| = 0$$

\noindent
{\bf Idea: } The tilted distribution 'looks' Gaussian and integrates same as $dP_{\frac{h}{\sqrt{n}},n}$ over compact sets.\\
\\
We will use the following lemma to show that, with Gaussian prior, a LAN family give asymptotically Gaussian posterior.\\
\\
{\bf Lemma 1.2. } Let $h \sim \mathcal{N}(0, \Gamma)$ with $\Gamma \succ 0$ and $Z|h \succ \mathcal{N}(Ah, \Sigma)$ with $\Sigma \succ 0$. Then $$h |Z=z \sim \mathcal{N}\left((\Gamma^{-1} + A^T\Sigma^{-1}A)^{-1}A^T \Sigma^{-1} z, \,\,(\Gamma^{-1} + A^T\Sigma^{-1}A)^{-1}\right)$$

\noindent
Define, for $K \succeq 0$, $\Gamma \succ 0$, the Gaussian distribution
$$G_{K, \Gamma}(\,.\,|z) = \mathcal{N}\left((K+\Gamma^{-1})^{-1}Kz,\,(K+\Gamma^{-1})^{-1} \right)$$

\noindent
{\bf Idea: } In LAN family, $Z_n = K^{-1}\Delta_n$ should give "Gaussian" posterior to shifts $\frac{h}{\sqrt{n}}$.\\
\\
{\bf Notation: } We use $\pi^{\Gamma, c}$ to denote the truncated Gaussian $\mathcal{N}(0, \Gamma)$ to $||h|| \le c$.\\
\\
{\bf Theorem 1.1 }  Let $\pi^{\Gamma, c}$ be the prior distribution distribution over $h$ and assume that data $X^n$ verify $X^n | h \sim P_{\frac{h}{\sqrt{n}},n}$. Denote $Z_n = K^{-1}\Delta_n(X^n)$ (LAN family), $P_n(\,.\,):= \int P_{\frac{h}{\sqrt{n}},n}(\,.\,) d\pi^{\Gamma, c}(h)$ the marginal distribution of $X^n$. Then, for all $\epsilon > 0$, there exist $C, N < + \infty$ such that for all $n \ge N$, $c \ge C$,
$$\int ||G_{K, \Gamma}(\,.\,|z_n(x^n)) - \pi^{\Gamma, c}(\,.\,|x^n)||_{TV} dP_n(x^n) \le \epsilon$$

\noindent
{\bf Remark: } The true posterior of a LAN family, under truncated Gaussian prior, is, on average, really close to a Gaussian distribution, conditioned on $Z_n = K^{-1}\Delta_n(x^n)$.\\

\section{Local asymptotic minimax theorem}

{\bf Definition 2.1. } A function $L:$ $\mathbb{R}^d\mapsto \mathbb{R}$ is quasi-convex if for all $\alpha \in \mathbb{R}$, the $\alpha$-sublevel set $ \{x:\,L(x) \le \alpha \}$ is convex.\\
\\
{\bf Example 2.1. } $L(x) = \frac{1}{2} ||x||_2^2 \land B $ is quasi-convex for any $B \in \mathbb{R}$.\\
\\
{\bf Lemma 2.1. (Anderson)} Let $L$ be symmetric and quasi-convex. Let $A \in \mathbb{R}^{d \times k}$ and $X \sim \mathcal{N}(\mu, \Sigma)$. Then:
$$\inf_{v \in \mathbb{R}^k} \mathbb{E}\left[L(AX-v)\right] = \mathbb{E}\left[L(A(X-\mu))\right] = \mathbb{E}\left[L(A \Sigma^{\frac{1}{2}}W)\right]$$
where $W \sim \mathcal{N}(0, I_k)$.\\
\\
{\bf Theorem 2.1. (Local asymptotic minimax)}\\ 
Let $L: \mathbb{R}^d \mapsto \mathbb{R}$ be quasi-convex, symmetric and bounded. Let $\{P_{\theta,n}\}$ be LAN at $\theta_0$ with precision $K_{\theta_0} \ge 0$. Then, with $W \sim \mathcal{N}(0, I_k)$,
$$\liminf_{c\infty} \liminf_{n \infty} \inf_{\hat{\theta}_n} \sup_{||h|| \le c,\,\, \theta = \theta_0 + \frac{h}{\sqrt{n}}} \mathbb{E}_{P_{\theta, n}} \left[ L(\sqrt{n} (\hat{\theta}_n - \theta)) \right] \ge \mathbb{E}\left[L(K_{\theta_0}^{-\frac{1}{2}} W)\right]$$ 

\noindent
{\bf Remark} Consider a quadratic mean differentiable family $\{P_{\theta}\}_{\theta \in \Theta}$ with Fisher information $I_{\theta_0}$ at parameter $\theta_0$. Then the theorem implies that: 
$$\liminf_{c \infty} \liminf_{n \infty} \inf_{\hat{\theta}_n} \int \mathbb{E}_{P^n_{\theta_0 + \frac{h}{\sqrt{n}}}} \left[ L(\sqrt{n} (\hat{\theta}_n(X_1,\dots,X_n) - \theta)) d\pi_c(h)\right] \ge \mathbb{E}[L(Z)]$$
with $Z \sim \mathcal{N}(0, I_{\theta_0}^{-1})$.\\
\\
{\bf Proof of Theorem 2.1. }\\
Without loss of generality, assume that L takes values in $[0,1]$ and $\theta_0 = 0$.\\
Observe that $$\sup_{||h|| \le c} \mathbb{E}_{P_{\frac{h}{\sqrt{n}},n}} \left[ L(\sqrt{n}(\hat{\theta}_n - \theta))\right] \ge \int \mathbb{E}_{P_{\frac{h}{\sqrt{n}},n}} \left[ L(\sqrt{n}\hat{\theta}_n - h)\right] d\pi(h)$$
where $\theta = \frac{h}{\sqrt{n}}$, for any $\pi$ with support in $\{||h|| \le c \}$.\\
\\
Consider $\pi := \pi^{\Gamma, c}$, prior of $h$, to be the normal distribution $\mathcal{N}(0, \Gamma)$, truncated to $\{||h|| \le c \}$ and denote the marginal distribution of $X^n$: $$\bar{P}_n(\,.\,) = \int P_{\frac{h}{\sqrt{n}},n} (\,.\,)d\pi^{\Gamma,c}(h)$$
Then, the left hand-side $(*)$ of the last inequality satisfies:
$$(*) \ge \int \mathbb{E}\left[ L(\sqrt{n}\hat{\theta}_n - h) \, | \, X^n = x^n\right] d\bar{P}_n(x^n) \ge \int \inf_{\hat{h}} \mathbb{E}\left[ L(\hat{h} - h) \, |\, X^n = x^n \right] d\bar{P}_n(x^n)$$
Using the previous notation $G_{K, \Gamma}$, we get:
$$(*) \ge \int \inf_{\hat{h}} \mathbb{E}_{G_{K, \Gamma}}\left[ L(\hat{h} - h) \,|\, x^n \right] d\bar{P}_n(x^n) - \int \sup_{h, \hat{h}} L(\hat{h} - h) \left( dG_{K, \Gamma}(h\,|\,x^n) - \pi(h\,|\,x^n)\right)d \bar{P}_n(x^n)$$

\noindent
Observe that:
$$\int \sup_{h, \hat{h}} L(\hat{h} - h) \left( dG_{K, \Gamma}(h\,|\,x^n) - \pi(h\,|\,x^n)\right)d \bar{P}(x^n) \le \int ||G_{K, \Gamma}(\,.\,|\,x^n) - \pi(\,.\,|x^n)||_{TV}d\bar{P}_n(x^n)$$
and that, by Theorem 1.1, the right-hand side of the last inequality is less than $\epsilon$, for any $\epsilon > 0$, $c$ appropriately chosen and $n$ sufficiently large.\\
\\
Moreover, by Anderson's lemma, we have:
$$\int \inf_{\hat{h}} \mathbb{E}_{G_{K, \Gamma}}\left[ L(\hat{h} - h) \,|\, x^n \right] d\bar{P}_n(x^n) \ge \int \mathbb{E}\left[L(\mathcal{N}(0, (K+\Gamma^{-1})^{-1}))\right] d\bar{P}_n(x^n)$$
Taking $\Gamma \rightarrow \infty$, we get:
$$(*) \ge \mathbb{E}\left[L(Z)\right] - \epsilon$$


\end{document}